change hard-coded perl path to variable
[platform/upstream/git.git] / git-p4.py
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
10 import sys
11 if sys.hexversion < 0x02040000:
12     # The limiter is the subprocess module
13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
14     sys.exit(1)
15 import os
16 import optparse
17 import marshal
18 import subprocess
19 import tempfile
20 import time
21 import platform
22 import re
23 import shutil
24 import stat
25
26 try:
27     from subprocess import CalledProcessError
28 except ImportError:
29     # from python2.7:subprocess.py
30     # Exception classes used by this module.
31     class CalledProcessError(Exception):
32         """This exception is raised when a process run by check_call() returns
33         a non-zero exit status.  The exit status will be stored in the
34         returncode attribute."""
35         def __init__(self, returncode, cmd):
36             self.returncode = returncode
37             self.cmd = cmd
38         def __str__(self):
39             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
40
41 verbose = False
42
43 # Only labels/tags matching this will be imported/exported
44 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
45
46 def p4_build_cmd(cmd):
47     """Build a suitable p4 command line.
48
49     This consolidates building and returning a p4 command line into one
50     location. It means that hooking into the environment, or other configuration
51     can be done more easily.
52     """
53     real_cmd = ["p4"]
54
55     user = gitConfig("git-p4.user")
56     if len(user) > 0:
57         real_cmd += ["-u",user]
58
59     password = gitConfig("git-p4.password")
60     if len(password) > 0:
61         real_cmd += ["-P", password]
62
63     port = gitConfig("git-p4.port")
64     if len(port) > 0:
65         real_cmd += ["-p", port]
66
67     host = gitConfig("git-p4.host")
68     if len(host) > 0:
69         real_cmd += ["-H", host]
70
71     client = gitConfig("git-p4.client")
72     if len(client) > 0:
73         real_cmd += ["-c", client]
74
75
76     if isinstance(cmd,basestring):
77         real_cmd = ' '.join(real_cmd) + ' ' + cmd
78     else:
79         real_cmd += cmd
80     return real_cmd
81
82 def chdir(path, is_client_path=False):
83     """Do chdir to the given path, and set the PWD environment
84        variable for use by P4.  It does not look at getcwd() output.
85        Since we're not using the shell, it is necessary to set the
86        PWD environment variable explicitly.
87
88        Normally, expand the path to force it to be absolute.  This
89        addresses the use of relative path names inside P4 settings,
90        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
91        as given; it looks for .p4config using PWD.
92
93        If is_client_path, the path was handed to us directly by p4,
94        and may be a symbolic link.  Do not call os.getcwd() in this
95        case, because it will cause p4 to think that PWD is not inside
96        the client path.
97        """
98
99     os.chdir(path)
100     if not is_client_path:
101         path = os.getcwd()
102     os.environ['PWD'] = path
103
104 def die(msg):
105     if verbose:
106         raise Exception(msg)
107     else:
108         sys.stderr.write(msg + "\n")
109         sys.exit(1)
110
111 def write_pipe(c, stdin):
112     if verbose:
113         sys.stderr.write('Writing pipe: %s\n' % str(c))
114
115     expand = isinstance(c,basestring)
116     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
117     pipe = p.stdin
118     val = pipe.write(stdin)
119     pipe.close()
120     if p.wait():
121         die('Command failed: %s' % str(c))
122
123     return val
124
125 def p4_write_pipe(c, stdin):
126     real_cmd = p4_build_cmd(c)
127     return write_pipe(real_cmd, stdin)
128
129 def read_pipe(c, ignore_error=False):
130     if verbose:
131         sys.stderr.write('Reading pipe: %s\n' % str(c))
132
133     expand = isinstance(c,basestring)
134     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
135     pipe = p.stdout
136     val = pipe.read()
137     if p.wait() and not ignore_error:
138         die('Command failed: %s' % str(c))
139
140     return val
141
142 def p4_read_pipe(c, ignore_error=False):
143     real_cmd = p4_build_cmd(c)
144     return read_pipe(real_cmd, ignore_error)
145
146 def read_pipe_lines(c):
147     if verbose:
148         sys.stderr.write('Reading pipe: %s\n' % str(c))
149
150     expand = isinstance(c, basestring)
151     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
152     pipe = p.stdout
153     val = pipe.readlines()
154     if pipe.close() or p.wait():
155         die('Command failed: %s' % str(c))
156
157     return val
158
159 def p4_read_pipe_lines(c):
160     """Specifically invoke p4 on the command supplied. """
161     real_cmd = p4_build_cmd(c)
162     return read_pipe_lines(real_cmd)
163
164 def p4_has_command(cmd):
165     """Ask p4 for help on this command.  If it returns an error, the
166        command does not exist in this version of p4."""
167     real_cmd = p4_build_cmd(["help", cmd])
168     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
169                                    stderr=subprocess.PIPE)
170     p.communicate()
171     return p.returncode == 0
172
173 def p4_has_move_command():
174     """See if the move command exists, that it supports -k, and that
175        it has not been administratively disabled.  The arguments
176        must be correct, but the filenames do not have to exist.  Use
177        ones with wildcards so even if they exist, it will fail."""
178
179     if not p4_has_command("move"):
180         return False
181     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
182     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
183     (out, err) = p.communicate()
184     # return code will be 1 in either case
185     if err.find("Invalid option") >= 0:
186         return False
187     if err.find("disabled") >= 0:
188         return False
189     # assume it failed because @... was invalid changelist
190     return True
191
192 def system(cmd):
193     expand = isinstance(cmd,basestring)
194     if verbose:
195         sys.stderr.write("executing %s\n" % str(cmd))
196     retcode = subprocess.call(cmd, shell=expand)
197     if retcode:
198         raise CalledProcessError(retcode, cmd)
199
200 def p4_system(cmd):
201     """Specifically invoke p4 as the system command. """
202     real_cmd = p4_build_cmd(cmd)
203     expand = isinstance(real_cmd, basestring)
204     retcode = subprocess.call(real_cmd, shell=expand)
205     if retcode:
206         raise CalledProcessError(retcode, real_cmd)
207
208 _p4_version_string = None
209 def p4_version_string():
210     """Read the version string, showing just the last line, which
211        hopefully is the interesting version bit.
212
213        $ p4 -V
214        Perforce - The Fast Software Configuration Management System.
215        Copyright 1995-2011 Perforce Software.  All rights reserved.
216        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
217     """
218     global _p4_version_string
219     if not _p4_version_string:
220         a = p4_read_pipe_lines(["-V"])
221         _p4_version_string = a[-1].rstrip()
222     return _p4_version_string
223
224 def p4_integrate(src, dest):
225     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
226
227 def p4_sync(f, *options):
228     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
229
230 def p4_add(f):
231     # forcibly add file names with wildcards
232     if wildcard_present(f):
233         p4_system(["add", "-f", f])
234     else:
235         p4_system(["add", f])
236
237 def p4_delete(f):
238     p4_system(["delete", wildcard_encode(f)])
239
240 def p4_edit(f):
241     p4_system(["edit", wildcard_encode(f)])
242
243 def p4_revert(f):
244     p4_system(["revert", wildcard_encode(f)])
245
246 def p4_reopen(type, f):
247     p4_system(["reopen", "-t", type, wildcard_encode(f)])
248
249 def p4_move(src, dest):
250     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
251
252 def p4_describe(change):
253     """Make sure it returns a valid result by checking for
254        the presence of field "time".  Return a dict of the
255        results."""
256
257     ds = p4CmdList(["describe", "-s", str(change)])
258     if len(ds) != 1:
259         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
260
261     d = ds[0]
262
263     if "p4ExitCode" in d:
264         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
265                                                       str(d)))
266     if "code" in d:
267         if d["code"] == "error":
268             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
269
270     if "time" not in d:
271         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
272
273     return d
274
275 #
276 # Canonicalize the p4 type and return a tuple of the
277 # base type, plus any modifiers.  See "p4 help filetypes"
278 # for a list and explanation.
279 #
280 def split_p4_type(p4type):
281
282     p4_filetypes_historical = {
283         "ctempobj": "binary+Sw",
284         "ctext": "text+C",
285         "cxtext": "text+Cx",
286         "ktext": "text+k",
287         "kxtext": "text+kx",
288         "ltext": "text+F",
289         "tempobj": "binary+FSw",
290         "ubinary": "binary+F",
291         "uresource": "resource+F",
292         "uxbinary": "binary+Fx",
293         "xbinary": "binary+x",
294         "xltext": "text+Fx",
295         "xtempobj": "binary+Swx",
296         "xtext": "text+x",
297         "xunicode": "unicode+x",
298         "xutf16": "utf16+x",
299     }
300     if p4type in p4_filetypes_historical:
301         p4type = p4_filetypes_historical[p4type]
302     mods = ""
303     s = p4type.split("+")
304     base = s[0]
305     mods = ""
306     if len(s) > 1:
307         mods = s[1]
308     return (base, mods)
309
310 #
311 # return the raw p4 type of a file (text, text+ko, etc)
312 #
313 def p4_type(f):
314     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
315     return results[0]['headType']
316
317 #
318 # Given a type base and modifier, return a regexp matching
319 # the keywords that can be expanded in the file
320 #
321 def p4_keywords_regexp_for_type(base, type_mods):
322     if base in ("text", "unicode", "binary"):
323         kwords = None
324         if "ko" in type_mods:
325             kwords = 'Id|Header'
326         elif "k" in type_mods:
327             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
328         else:
329             return None
330         pattern = r"""
331             \$              # Starts with a dollar, followed by...
332             (%s)            # one of the keywords, followed by...
333             (:[^$\n]+)?     # possibly an old expansion, followed by...
334             \$              # another dollar
335             """ % kwords
336         return pattern
337     else:
338         return None
339
340 #
341 # Given a file, return a regexp matching the possible
342 # RCS keywords that will be expanded, or None for files
343 # with kw expansion turned off.
344 #
345 def p4_keywords_regexp_for_file(file):
346     if not os.path.exists(file):
347         return None
348     else:
349         (type_base, type_mods) = split_p4_type(p4_type(file))
350         return p4_keywords_regexp_for_type(type_base, type_mods)
351
352 def setP4ExecBit(file, mode):
353     # Reopens an already open file and changes the execute bit to match
354     # the execute bit setting in the passed in mode.
355
356     p4Type = "+x"
357
358     if not isModeExec(mode):
359         p4Type = getP4OpenedType(file)
360         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
361         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
362         if p4Type[-1] == "+":
363             p4Type = p4Type[0:-1]
364
365     p4_reopen(p4Type, file)
366
367 def getP4OpenedType(file):
368     # Returns the perforce file type for the given file.
369
370     result = p4_read_pipe(["opened", wildcard_encode(file)])
371     match = re.match(".*\((.+)\)\r?$", result)
372     if match:
373         return match.group(1)
374     else:
375         die("Could not determine file type for %s (result: '%s')" % (file, result))
376
377 # Return the set of all p4 labels
378 def getP4Labels(depotPaths):
379     labels = set()
380     if isinstance(depotPaths,basestring):
381         depotPaths = [depotPaths]
382
383     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
384         label = l['label']
385         labels.add(label)
386
387     return labels
388
389 # Return the set of all git tags
390 def getGitTags():
391     gitTags = set()
392     for line in read_pipe_lines(["git", "tag"]):
393         tag = line.strip()
394         gitTags.add(tag)
395     return gitTags
396
397 def diffTreePattern():
398     # This is a simple generator for the diff tree regex pattern. This could be
399     # a class variable if this and parseDiffTreeEntry were a part of a class.
400     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
401     while True:
402         yield pattern
403
404 def parseDiffTreeEntry(entry):
405     """Parses a single diff tree entry into its component elements.
406
407     See git-diff-tree(1) manpage for details about the format of the diff
408     output. This method returns a dictionary with the following elements:
409
410     src_mode - The mode of the source file
411     dst_mode - The mode of the destination file
412     src_sha1 - The sha1 for the source file
413     dst_sha1 - The sha1 fr the destination file
414     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
415     status_score - The score for the status (applicable for 'C' and 'R'
416                    statuses). This is None if there is no score.
417     src - The path for the source file.
418     dst - The path for the destination file. This is only present for
419           copy or renames. If it is not present, this is None.
420
421     If the pattern is not matched, None is returned."""
422
423     match = diffTreePattern().next().match(entry)
424     if match:
425         return {
426             'src_mode': match.group(1),
427             'dst_mode': match.group(2),
428             'src_sha1': match.group(3),
429             'dst_sha1': match.group(4),
430             'status': match.group(5),
431             'status_score': match.group(6),
432             'src': match.group(7),
433             'dst': match.group(10)
434         }
435     return None
436
437 def isModeExec(mode):
438     # Returns True if the given git mode represents an executable file,
439     # otherwise False.
440     return mode[-3:] == "755"
441
442 def isModeExecChanged(src_mode, dst_mode):
443     return isModeExec(src_mode) != isModeExec(dst_mode)
444
445 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
446
447     if isinstance(cmd,basestring):
448         cmd = "-G " + cmd
449         expand = True
450     else:
451         cmd = ["-G"] + cmd
452         expand = False
453
454     cmd = p4_build_cmd(cmd)
455     if verbose:
456         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
457
458     # Use a temporary file to avoid deadlocks without
459     # subprocess.communicate(), which would put another copy
460     # of stdout into memory.
461     stdin_file = None
462     if stdin is not None:
463         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
464         if isinstance(stdin,basestring):
465             stdin_file.write(stdin)
466         else:
467             for i in stdin:
468                 stdin_file.write(i + '\n')
469         stdin_file.flush()
470         stdin_file.seek(0)
471
472     p4 = subprocess.Popen(cmd,
473                           shell=expand,
474                           stdin=stdin_file,
475                           stdout=subprocess.PIPE)
476
477     result = []
478     try:
479         while True:
480             entry = marshal.load(p4.stdout)
481             if cb is not None:
482                 cb(entry)
483             else:
484                 result.append(entry)
485     except EOFError:
486         pass
487     exitCode = p4.wait()
488     if exitCode != 0:
489         entry = {}
490         entry["p4ExitCode"] = exitCode
491         result.append(entry)
492
493     return result
494
495 def p4Cmd(cmd):
496     list = p4CmdList(cmd)
497     result = {}
498     for entry in list:
499         result.update(entry)
500     return result;
501
502 def p4Where(depotPath):
503     if not depotPath.endswith("/"):
504         depotPath += "/"
505     depotPath = depotPath + "..."
506     outputList = p4CmdList(["where", depotPath])
507     output = None
508     for entry in outputList:
509         if "depotFile" in entry:
510             if entry["depotFile"] == depotPath:
511                 output = entry
512                 break
513         elif "data" in entry:
514             data = entry.get("data")
515             space = data.find(" ")
516             if data[:space] == depotPath:
517                 output = entry
518                 break
519     if output == None:
520         return ""
521     if output["code"] == "error":
522         return ""
523     clientPath = ""
524     if "path" in output:
525         clientPath = output.get("path")
526     elif "data" in output:
527         data = output.get("data")
528         lastSpace = data.rfind(" ")
529         clientPath = data[lastSpace + 1:]
530
531     if clientPath.endswith("..."):
532         clientPath = clientPath[:-3]
533     return clientPath
534
535 def currentGitBranch():
536     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
537
538 def isValidGitDir(path):
539     if (os.path.exists(path + "/HEAD")
540         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
541         return True;
542     return False
543
544 def parseRevision(ref):
545     return read_pipe("git rev-parse %s" % ref).strip()
546
547 def branchExists(ref):
548     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
549                      ignore_error=True)
550     return len(rev) > 0
551
552 def extractLogMessageFromGitCommit(commit):
553     logMessage = ""
554
555     ## fixme: title is first line of commit, not 1st paragraph.
556     foundTitle = False
557     for log in read_pipe_lines("git cat-file commit %s" % commit):
558        if not foundTitle:
559            if len(log) == 1:
560                foundTitle = True
561            continue
562
563        logMessage += log
564     return logMessage
565
566 def extractSettingsGitLog(log):
567     values = {}
568     for line in log.split("\n"):
569         line = line.strip()
570         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
571         if not m:
572             continue
573
574         assignments = m.group(1).split (':')
575         for a in assignments:
576             vals = a.split ('=')
577             key = vals[0].strip()
578             val = ('='.join (vals[1:])).strip()
579             if val.endswith ('\"') and val.startswith('"'):
580                 val = val[1:-1]
581
582             values[key] = val
583
584     paths = values.get("depot-paths")
585     if not paths:
586         paths = values.get("depot-path")
587     if paths:
588         values['depot-paths'] = paths.split(',')
589     return values
590
591 def gitBranchExists(branch):
592     proc = subprocess.Popen(["git", "rev-parse", branch],
593                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
594     return proc.wait() == 0;
595
596 _gitConfig = {}
597
598 def gitConfig(key):
599     if not _gitConfig.has_key(key):
600         cmd = [ "git", "config", key ]
601         s = read_pipe(cmd, ignore_error=True)
602         _gitConfig[key] = s.strip()
603     return _gitConfig[key]
604
605 def gitConfigBool(key):
606     """Return a bool, using git config --bool.  It is True only if the
607        variable is set to true, and False if set to false or not present
608        in the config."""
609
610     if not _gitConfig.has_key(key):
611         cmd = [ "git", "config", "--bool", key ]
612         s = read_pipe(cmd, ignore_error=True)
613         v = s.strip()
614         _gitConfig[key] = v == "true"
615     return _gitConfig[key]
616
617 def gitConfigList(key):
618     if not _gitConfig.has_key(key):
619         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
620         _gitConfig[key] = s.strip().split(os.linesep)
621     return _gitConfig[key]
622
623 def p4BranchesInGit(branchesAreInRemotes=True):
624     """Find all the branches whose names start with "p4/", looking
625        in remotes or heads as specified by the argument.  Return
626        a dictionary of { branch: revision } for each one found.
627        The branch names are the short names, without any
628        "p4/" prefix."""
629
630     branches = {}
631
632     cmdline = "git rev-parse --symbolic "
633     if branchesAreInRemotes:
634         cmdline += "--remotes"
635     else:
636         cmdline += "--branches"
637
638     for line in read_pipe_lines(cmdline):
639         line = line.strip()
640
641         # only import to p4/
642         if not line.startswith('p4/'):
643             continue
644         # special symbolic ref to p4/master
645         if line == "p4/HEAD":
646             continue
647
648         # strip off p4/ prefix
649         branch = line[len("p4/"):]
650
651         branches[branch] = parseRevision(line)
652
653     return branches
654
655 def branch_exists(branch):
656     """Make sure that the given ref name really exists."""
657
658     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
659     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
660     out, _ = p.communicate()
661     if p.returncode:
662         return False
663     # expect exactly one line of output: the branch name
664     return out.rstrip() == branch
665
666 def findUpstreamBranchPoint(head = "HEAD"):
667     branches = p4BranchesInGit()
668     # map from depot-path to branch name
669     branchByDepotPath = {}
670     for branch in branches.keys():
671         tip = branches[branch]
672         log = extractLogMessageFromGitCommit(tip)
673         settings = extractSettingsGitLog(log)
674         if settings.has_key("depot-paths"):
675             paths = ",".join(settings["depot-paths"])
676             branchByDepotPath[paths] = "remotes/p4/" + branch
677
678     settings = None
679     parent = 0
680     while parent < 65535:
681         commit = head + "~%s" % parent
682         log = extractLogMessageFromGitCommit(commit)
683         settings = extractSettingsGitLog(log)
684         if settings.has_key("depot-paths"):
685             paths = ",".join(settings["depot-paths"])
686             if branchByDepotPath.has_key(paths):
687                 return [branchByDepotPath[paths], settings]
688
689         parent = parent + 1
690
691     return ["", settings]
692
693 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
694     if not silent:
695         print ("Creating/updating branch(es) in %s based on origin branch(es)"
696                % localRefPrefix)
697
698     originPrefix = "origin/p4/"
699
700     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
701         line = line.strip()
702         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
703             continue
704
705         headName = line[len(originPrefix):]
706         remoteHead = localRefPrefix + headName
707         originHead = line
708
709         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
710         if (not original.has_key('depot-paths')
711             or not original.has_key('change')):
712             continue
713
714         update = False
715         if not gitBranchExists(remoteHead):
716             if verbose:
717                 print "creating %s" % remoteHead
718             update = True
719         else:
720             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
721             if settings.has_key('change') > 0:
722                 if settings['depot-paths'] == original['depot-paths']:
723                     originP4Change = int(original['change'])
724                     p4Change = int(settings['change'])
725                     if originP4Change > p4Change:
726                         print ("%s (%s) is newer than %s (%s). "
727                                "Updating p4 branch from origin."
728                                % (originHead, originP4Change,
729                                   remoteHead, p4Change))
730                         update = True
731                 else:
732                     print ("Ignoring: %s was imported from %s while "
733                            "%s was imported from %s"
734                            % (originHead, ','.join(original['depot-paths']),
735                               remoteHead, ','.join(settings['depot-paths'])))
736
737         if update:
738             system("git update-ref %s %s" % (remoteHead, originHead))
739
740 def originP4BranchesExist():
741         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
742
743 def p4ChangesForPaths(depotPaths, changeRange):
744     assert depotPaths
745     cmd = ['changes']
746     for p in depotPaths:
747         cmd += ["%s...%s" % (p, changeRange)]
748     output = p4_read_pipe_lines(cmd)
749
750     changes = {}
751     for line in output:
752         changeNum = int(line.split(" ")[1])
753         changes[changeNum] = True
754
755     changelist = changes.keys()
756     changelist.sort()
757     return changelist
758
759 def p4PathStartsWith(path, prefix):
760     # This method tries to remedy a potential mixed-case issue:
761     #
762     # If UserA adds  //depot/DirA/file1
763     # and UserB adds //depot/dira/file2
764     #
765     # we may or may not have a problem. If you have core.ignorecase=true,
766     # we treat DirA and dira as the same directory
767     if gitConfigBool("core.ignorecase"):
768         return path.lower().startswith(prefix.lower())
769     return path.startswith(prefix)
770
771 def getClientSpec():
772     """Look at the p4 client spec, create a View() object that contains
773        all the mappings, and return it."""
774
775     specList = p4CmdList("client -o")
776     if len(specList) != 1:
777         die('Output from "client -o" is %d lines, expecting 1' %
778             len(specList))
779
780     # dictionary of all client parameters
781     entry = specList[0]
782
783     # the //client/ name
784     client_name = entry["Client"]
785
786     # just the keys that start with "View"
787     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
788
789     # hold this new View
790     view = View(client_name)
791
792     # append the lines, in order, to the view
793     for view_num in range(len(view_keys)):
794         k = "View%d" % view_num
795         if k not in view_keys:
796             die("Expected view key %s missing" % k)
797         view.append(entry[k])
798
799     return view
800
801 def getClientRoot():
802     """Grab the client directory."""
803
804     output = p4CmdList("client -o")
805     if len(output) != 1:
806         die('Output from "client -o" is %d lines, expecting 1' % len(output))
807
808     entry = output[0]
809     if "Root" not in entry:
810         die('Client has no "Root"')
811
812     return entry["Root"]
813
814 #
815 # P4 wildcards are not allowed in filenames.  P4 complains
816 # if you simply add them, but you can force it with "-f", in
817 # which case it translates them into %xx encoding internally.
818 #
819 def wildcard_decode(path):
820     # Search for and fix just these four characters.  Do % last so
821     # that fixing it does not inadvertently create new %-escapes.
822     # Cannot have * in a filename in windows; untested as to
823     # what p4 would do in such a case.
824     if not platform.system() == "Windows":
825         path = path.replace("%2A", "*")
826     path = path.replace("%23", "#") \
827                .replace("%40", "@") \
828                .replace("%25", "%")
829     return path
830
831 def wildcard_encode(path):
832     # do % first to avoid double-encoding the %s introduced here
833     path = path.replace("%", "%25") \
834                .replace("*", "%2A") \
835                .replace("#", "%23") \
836                .replace("@", "%40")
837     return path
838
839 def wildcard_present(path):
840     m = re.search("[*#@%]", path)
841     return m is not None
842
843 class Command:
844     def __init__(self):
845         self.usage = "usage: %prog [options]"
846         self.needsGit = True
847         self.verbose = False
848
849 class P4UserMap:
850     def __init__(self):
851         self.userMapFromPerforceServer = False
852         self.myP4UserId = None
853
854     def p4UserId(self):
855         if self.myP4UserId:
856             return self.myP4UserId
857
858         results = p4CmdList("user -o")
859         for r in results:
860             if r.has_key('User'):
861                 self.myP4UserId = r['User']
862                 return r['User']
863         die("Could not find your p4 user id")
864
865     def p4UserIsMe(self, p4User):
866         # return True if the given p4 user is actually me
867         me = self.p4UserId()
868         if not p4User or p4User != me:
869             return False
870         else:
871             return True
872
873     def getUserCacheFilename(self):
874         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
875         return home + "/.gitp4-usercache.txt"
876
877     def getUserMapFromPerforceServer(self):
878         if self.userMapFromPerforceServer:
879             return
880         self.users = {}
881         self.emails = {}
882
883         for output in p4CmdList("users"):
884             if not output.has_key("User"):
885                 continue
886             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
887             self.emails[output["Email"]] = output["User"]
888
889
890         s = ''
891         for (key, val) in self.users.items():
892             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
893
894         open(self.getUserCacheFilename(), "wb").write(s)
895         self.userMapFromPerforceServer = True
896
897     def loadUserMapFromCache(self):
898         self.users = {}
899         self.userMapFromPerforceServer = False
900         try:
901             cache = open(self.getUserCacheFilename(), "rb")
902             lines = cache.readlines()
903             cache.close()
904             for line in lines:
905                 entry = line.strip().split("\t")
906                 self.users[entry[0]] = entry[1]
907         except IOError:
908             self.getUserMapFromPerforceServer()
909
910 class P4Debug(Command):
911     def __init__(self):
912         Command.__init__(self)
913         self.options = []
914         self.description = "A tool to debug the output of p4 -G."
915         self.needsGit = False
916
917     def run(self, args):
918         j = 0
919         for output in p4CmdList(args):
920             print 'Element: %d' % j
921             j += 1
922             print output
923         return True
924
925 class P4RollBack(Command):
926     def __init__(self):
927         Command.__init__(self)
928         self.options = [
929             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
930         ]
931         self.description = "A tool to debug the multi-branch import. Don't use :)"
932         self.rollbackLocalBranches = False
933
934     def run(self, args):
935         if len(args) != 1:
936             return False
937         maxChange = int(args[0])
938
939         if "p4ExitCode" in p4Cmd("changes -m 1"):
940             die("Problems executing p4");
941
942         if self.rollbackLocalBranches:
943             refPrefix = "refs/heads/"
944             lines = read_pipe_lines("git rev-parse --symbolic --branches")
945         else:
946             refPrefix = "refs/remotes/"
947             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
948
949         for line in lines:
950             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
951                 line = line.strip()
952                 ref = refPrefix + line
953                 log = extractLogMessageFromGitCommit(ref)
954                 settings = extractSettingsGitLog(log)
955
956                 depotPaths = settings['depot-paths']
957                 change = settings['change']
958
959                 changed = False
960
961                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
962                                                            for p in depotPaths]))) == 0:
963                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
964                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
965                     continue
966
967                 while change and int(change) > maxChange:
968                     changed = True
969                     if self.verbose:
970                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
971                     system("git update-ref %s \"%s^\"" % (ref, ref))
972                     log = extractLogMessageFromGitCommit(ref)
973                     settings =  extractSettingsGitLog(log)
974
975
976                     depotPaths = settings['depot-paths']
977                     change = settings['change']
978
979                 if changed:
980                     print "%s rewound to %s" % (ref, change)
981
982         return True
983
984 class P4Submit(Command, P4UserMap):
985
986     conflict_behavior_choices = ("ask", "skip", "quit")
987
988     def __init__(self):
989         Command.__init__(self)
990         P4UserMap.__init__(self)
991         self.options = [
992                 optparse.make_option("--origin", dest="origin"),
993                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
994                 # preserve the user, requires relevant p4 permissions
995                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
996                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
997                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
998                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
999                 optparse.make_option("--conflict", dest="conflict_behavior",
1000                                      choices=self.conflict_behavior_choices),
1001                 optparse.make_option("--branch", dest="branch"),
1002         ]
1003         self.description = "Submit changes from git to the perforce depot."
1004         self.usage += " [name of git branch to submit into perforce depot]"
1005         self.origin = ""
1006         self.detectRenames = False
1007         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1008         self.dry_run = False
1009         self.prepare_p4_only = False
1010         self.conflict_behavior = None
1011         self.isWindows = (platform.system() == "Windows")
1012         self.exportLabels = False
1013         self.p4HasMoveCommand = p4_has_move_command()
1014         self.branch = None
1015
1016     def check(self):
1017         if len(p4CmdList("opened ...")) > 0:
1018             die("You have files opened with perforce! Close them before starting the sync.")
1019
1020     def separate_jobs_from_description(self, message):
1021         """Extract and return a possible Jobs field in the commit
1022            message.  It goes into a separate section in the p4 change
1023            specification.
1024
1025            A jobs line starts with "Jobs:" and looks like a new field
1026            in a form.  Values are white-space separated on the same
1027            line or on following lines that start with a tab.
1028
1029            This does not parse and extract the full git commit message
1030            like a p4 form.  It just sees the Jobs: line as a marker
1031            to pass everything from then on directly into the p4 form,
1032            but outside the description section.
1033
1034            Return a tuple (stripped log message, jobs string)."""
1035
1036         m = re.search(r'^Jobs:', message, re.MULTILINE)
1037         if m is None:
1038             return (message, None)
1039
1040         jobtext = message[m.start():]
1041         stripped_message = message[:m.start()].rstrip()
1042         return (stripped_message, jobtext)
1043
1044     def prepareLogMessage(self, template, message, jobs):
1045         """Edits the template returned from "p4 change -o" to insert
1046            the message in the Description field, and the jobs text in
1047            the Jobs field."""
1048         result = ""
1049
1050         inDescriptionSection = False
1051
1052         for line in template.split("\n"):
1053             if line.startswith("#"):
1054                 result += line + "\n"
1055                 continue
1056
1057             if inDescriptionSection:
1058                 if line.startswith("Files:") or line.startswith("Jobs:"):
1059                     inDescriptionSection = False
1060                     # insert Jobs section
1061                     if jobs:
1062                         result += jobs + "\n"
1063                 else:
1064                     continue
1065             else:
1066                 if line.startswith("Description:"):
1067                     inDescriptionSection = True
1068                     line += "\n"
1069                     for messageLine in message.split("\n"):
1070                         line += "\t" + messageLine + "\n"
1071
1072             result += line + "\n"
1073
1074         return result
1075
1076     def patchRCSKeywords(self, file, pattern):
1077         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1078         (handle, outFileName) = tempfile.mkstemp(dir='.')
1079         try:
1080             outFile = os.fdopen(handle, "w+")
1081             inFile = open(file, "r")
1082             regexp = re.compile(pattern, re.VERBOSE)
1083             for line in inFile.readlines():
1084                 line = regexp.sub(r'$\1$', line)
1085                 outFile.write(line)
1086             inFile.close()
1087             outFile.close()
1088             # Forcibly overwrite the original file
1089             os.unlink(file)
1090             shutil.move(outFileName, file)
1091         except:
1092             # cleanup our temporary file
1093             os.unlink(outFileName)
1094             print "Failed to strip RCS keywords in %s" % file
1095             raise
1096
1097         print "Patched up RCS keywords in %s" % file
1098
1099     def p4UserForCommit(self,id):
1100         # Return the tuple (perforce user,git email) for a given git commit id
1101         self.getUserMapFromPerforceServer()
1102         gitEmail = read_pipe(["git", "log", "--max-count=1",
1103                               "--format=%ae", id])
1104         gitEmail = gitEmail.strip()
1105         if not self.emails.has_key(gitEmail):
1106             return (None,gitEmail)
1107         else:
1108             return (self.emails[gitEmail],gitEmail)
1109
1110     def checkValidP4Users(self,commits):
1111         # check if any git authors cannot be mapped to p4 users
1112         for id in commits:
1113             (user,email) = self.p4UserForCommit(id)
1114             if not user:
1115                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1116                 if gitConfigBool("git-p4.allowMissingP4Users"):
1117                     print "%s" % msg
1118                 else:
1119                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1120
1121     def lastP4Changelist(self):
1122         # Get back the last changelist number submitted in this client spec. This
1123         # then gets used to patch up the username in the change. If the same
1124         # client spec is being used by multiple processes then this might go
1125         # wrong.
1126         results = p4CmdList("client -o")        # find the current client
1127         client = None
1128         for r in results:
1129             if r.has_key('Client'):
1130                 client = r['Client']
1131                 break
1132         if not client:
1133             die("could not get client spec")
1134         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1135         for r in results:
1136             if r.has_key('change'):
1137                 return r['change']
1138         die("Could not get changelist number for last submit - cannot patch up user details")
1139
1140     def modifyChangelistUser(self, changelist, newUser):
1141         # fixup the user field of a changelist after it has been submitted.
1142         changes = p4CmdList("change -o %s" % changelist)
1143         if len(changes) != 1:
1144             die("Bad output from p4 change modifying %s to user %s" %
1145                 (changelist, newUser))
1146
1147         c = changes[0]
1148         if c['User'] == newUser: return   # nothing to do
1149         c['User'] = newUser
1150         input = marshal.dumps(c)
1151
1152         result = p4CmdList("change -f -i", stdin=input)
1153         for r in result:
1154             if r.has_key('code'):
1155                 if r['code'] == 'error':
1156                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1157             if r.has_key('data'):
1158                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1159                 return
1160         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1161
1162     def canChangeChangelists(self):
1163         # check to see if we have p4 admin or super-user permissions, either of
1164         # which are required to modify changelists.
1165         results = p4CmdList(["protects", self.depotPath])
1166         for r in results:
1167             if r.has_key('perm'):
1168                 if r['perm'] == 'admin':
1169                     return 1
1170                 if r['perm'] == 'super':
1171                     return 1
1172         return 0
1173
1174     def prepareSubmitTemplate(self):
1175         """Run "p4 change -o" to grab a change specification template.
1176            This does not use "p4 -G", as it is nice to keep the submission
1177            template in original order, since a human might edit it.
1178
1179            Remove lines in the Files section that show changes to files
1180            outside the depot path we're committing into."""
1181
1182         template = ""
1183         inFilesSection = False
1184         for line in p4_read_pipe_lines(['change', '-o']):
1185             if line.endswith("\r\n"):
1186                 line = line[:-2] + "\n"
1187             if inFilesSection:
1188                 if line.startswith("\t"):
1189                     # path starts and ends with a tab
1190                     path = line[1:]
1191                     lastTab = path.rfind("\t")
1192                     if lastTab != -1:
1193                         path = path[:lastTab]
1194                         if not p4PathStartsWith(path, self.depotPath):
1195                             continue
1196                 else:
1197                     inFilesSection = False
1198             else:
1199                 if line.startswith("Files:"):
1200                     inFilesSection = True
1201
1202             template += line
1203
1204         return template
1205
1206     def edit_template(self, template_file):
1207         """Invoke the editor to let the user change the submission
1208            message.  Return true if okay to continue with the submit."""
1209
1210         # if configured to skip the editing part, just submit
1211         if gitConfigBool("git-p4.skipSubmitEdit"):
1212             return True
1213
1214         # look at the modification time, to check later if the user saved
1215         # the file
1216         mtime = os.stat(template_file).st_mtime
1217
1218         # invoke the editor
1219         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1220             editor = os.environ.get("P4EDITOR")
1221         else:
1222             editor = read_pipe("git var GIT_EDITOR").strip()
1223         system([editor, template_file])
1224
1225         # If the file was not saved, prompt to see if this patch should
1226         # be skipped.  But skip this verification step if configured so.
1227         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1228             return True
1229
1230         # modification time updated means user saved the file
1231         if os.stat(template_file).st_mtime > mtime:
1232             return True
1233
1234         while True:
1235             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1236             if response == 'y':
1237                 return True
1238             if response == 'n':
1239                 return False
1240
1241     def applyCommit(self, id):
1242         """Apply one commit, return True if it succeeded."""
1243
1244         print "Applying", read_pipe(["git", "show", "-s",
1245                                      "--format=format:%h %s", id])
1246
1247         (p4User, gitEmail) = self.p4UserForCommit(id)
1248
1249         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1250         filesToAdd = set()
1251         filesToDelete = set()
1252         editedFiles = set()
1253         pureRenameCopy = set()
1254         filesToChangeExecBit = {}
1255
1256         for line in diff:
1257             diff = parseDiffTreeEntry(line)
1258             modifier = diff['status']
1259             path = diff['src']
1260             if modifier == "M":
1261                 p4_edit(path)
1262                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1263                     filesToChangeExecBit[path] = diff['dst_mode']
1264                 editedFiles.add(path)
1265             elif modifier == "A":
1266                 filesToAdd.add(path)
1267                 filesToChangeExecBit[path] = diff['dst_mode']
1268                 if path in filesToDelete:
1269                     filesToDelete.remove(path)
1270             elif modifier == "D":
1271                 filesToDelete.add(path)
1272                 if path in filesToAdd:
1273                     filesToAdd.remove(path)
1274             elif modifier == "C":
1275                 src, dest = diff['src'], diff['dst']
1276                 p4_integrate(src, dest)
1277                 pureRenameCopy.add(dest)
1278                 if diff['src_sha1'] != diff['dst_sha1']:
1279                     p4_edit(dest)
1280                     pureRenameCopy.discard(dest)
1281                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1282                     p4_edit(dest)
1283                     pureRenameCopy.discard(dest)
1284                     filesToChangeExecBit[dest] = diff['dst_mode']
1285                 if self.isWindows:
1286                     # turn off read-only attribute
1287                     os.chmod(dest, stat.S_IWRITE)
1288                 os.unlink(dest)
1289                 editedFiles.add(dest)
1290             elif modifier == "R":
1291                 src, dest = diff['src'], diff['dst']
1292                 if self.p4HasMoveCommand:
1293                     p4_edit(src)        # src must be open before move
1294                     p4_move(src, dest)  # opens for (move/delete, move/add)
1295                 else:
1296                     p4_integrate(src, dest)
1297                     if diff['src_sha1'] != diff['dst_sha1']:
1298                         p4_edit(dest)
1299                     else:
1300                         pureRenameCopy.add(dest)
1301                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1302                     if not self.p4HasMoveCommand:
1303                         p4_edit(dest)   # with move: already open, writable
1304                     filesToChangeExecBit[dest] = diff['dst_mode']
1305                 if not self.p4HasMoveCommand:
1306                     if self.isWindows:
1307                         os.chmod(dest, stat.S_IWRITE)
1308                     os.unlink(dest)
1309                     filesToDelete.add(src)
1310                 editedFiles.add(dest)
1311             else:
1312                 die("unknown modifier %s for %s" % (modifier, path))
1313
1314         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1315         patchcmd = diffcmd + " | git apply "
1316         tryPatchCmd = patchcmd + "--check -"
1317         applyPatchCmd = patchcmd + "--check --apply -"
1318         patch_succeeded = True
1319
1320         if os.system(tryPatchCmd) != 0:
1321             fixed_rcs_keywords = False
1322             patch_succeeded = False
1323             print "Unfortunately applying the change failed!"
1324
1325             # Patch failed, maybe it's just RCS keyword woes. Look through
1326             # the patch to see if that's possible.
1327             if gitConfigBool("git-p4.attemptRCSCleanup"):
1328                 file = None
1329                 pattern = None
1330                 kwfiles = {}
1331                 for file in editedFiles | filesToDelete:
1332                     # did this file's delta contain RCS keywords?
1333                     pattern = p4_keywords_regexp_for_file(file)
1334
1335                     if pattern:
1336                         # this file is a possibility...look for RCS keywords.
1337                         regexp = re.compile(pattern, re.VERBOSE)
1338                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1339                             if regexp.search(line):
1340                                 if verbose:
1341                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1342                                 kwfiles[file] = pattern
1343                                 break
1344
1345                 for file in kwfiles:
1346                     if verbose:
1347                         print "zapping %s with %s" % (line,pattern)
1348                     # File is being deleted, so not open in p4.  Must
1349                     # disable the read-only bit on windows.
1350                     if self.isWindows and file not in editedFiles:
1351                         os.chmod(file, stat.S_IWRITE)
1352                     self.patchRCSKeywords(file, kwfiles[file])
1353                     fixed_rcs_keywords = True
1354
1355             if fixed_rcs_keywords:
1356                 print "Retrying the patch with RCS keywords cleaned up"
1357                 if os.system(tryPatchCmd) == 0:
1358                     patch_succeeded = True
1359
1360         if not patch_succeeded:
1361             for f in editedFiles:
1362                 p4_revert(f)
1363             return False
1364
1365         #
1366         # Apply the patch for real, and do add/delete/+x handling.
1367         #
1368         system(applyPatchCmd)
1369
1370         for f in filesToAdd:
1371             p4_add(f)
1372         for f in filesToDelete:
1373             p4_revert(f)
1374             p4_delete(f)
1375
1376         # Set/clear executable bits
1377         for f in filesToChangeExecBit.keys():
1378             mode = filesToChangeExecBit[f]
1379             setP4ExecBit(f, mode)
1380
1381         #
1382         # Build p4 change description, starting with the contents
1383         # of the git commit message.
1384         #
1385         logMessage = extractLogMessageFromGitCommit(id)
1386         logMessage = logMessage.strip()
1387         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1388
1389         template = self.prepareSubmitTemplate()
1390         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1391
1392         if self.preserveUser:
1393            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1394
1395         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1396             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1397             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1398             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1399
1400         separatorLine = "######## everything below this line is just the diff #######\n"
1401
1402         # diff
1403         if os.environ.has_key("P4DIFF"):
1404             del(os.environ["P4DIFF"])
1405         diff = ""
1406         for editedFile in editedFiles:
1407             diff += p4_read_pipe(['diff', '-du',
1408                                   wildcard_encode(editedFile)])
1409
1410         # new file diff
1411         newdiff = ""
1412         for newFile in filesToAdd:
1413             newdiff += "==== new file ====\n"
1414             newdiff += "--- /dev/null\n"
1415             newdiff += "+++ %s\n" % newFile
1416             f = open(newFile, "r")
1417             for line in f.readlines():
1418                 newdiff += "+" + line
1419             f.close()
1420
1421         # change description file: submitTemplate, separatorLine, diff, newdiff
1422         (handle, fileName) = tempfile.mkstemp()
1423         tmpFile = os.fdopen(handle, "w+")
1424         if self.isWindows:
1425             submitTemplate = submitTemplate.replace("\n", "\r\n")
1426             separatorLine = separatorLine.replace("\n", "\r\n")
1427             newdiff = newdiff.replace("\n", "\r\n")
1428         tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1429         tmpFile.close()
1430
1431         if self.prepare_p4_only:
1432             #
1433             # Leave the p4 tree prepared, and the submit template around
1434             # and let the user decide what to do next
1435             #
1436             print
1437             print "P4 workspace prepared for submission."
1438             print "To submit or revert, go to client workspace"
1439             print "  " + self.clientPath
1440             print
1441             print "To submit, use \"p4 submit\" to write a new description,"
1442             print "or \"p4 submit -i %s\" to use the one prepared by" \
1443                   " \"git p4\"." % fileName
1444             print "You can delete the file \"%s\" when finished." % fileName
1445
1446             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1447                 print "To preserve change ownership by user %s, you must\n" \
1448                       "do \"p4 change -f <change>\" after submitting and\n" \
1449                       "edit the User field."
1450             if pureRenameCopy:
1451                 print "After submitting, renamed files must be re-synced."
1452                 print "Invoke \"p4 sync -f\" on each of these files:"
1453                 for f in pureRenameCopy:
1454                     print "  " + f
1455
1456             print
1457             print "To revert the changes, use \"p4 revert ...\", and delete"
1458             print "the submit template file \"%s\"" % fileName
1459             if filesToAdd:
1460                 print "Since the commit adds new files, they must be deleted:"
1461                 for f in filesToAdd:
1462                     print "  " + f
1463             print
1464             return True
1465
1466         #
1467         # Let the user edit the change description, then submit it.
1468         #
1469         if self.edit_template(fileName):
1470             # read the edited message and submit
1471             ret = True
1472             tmpFile = open(fileName, "rb")
1473             message = tmpFile.read()
1474             tmpFile.close()
1475             submitTemplate = message[:message.index(separatorLine)]
1476             if self.isWindows:
1477                 submitTemplate = submitTemplate.replace("\r\n", "\n")
1478             p4_write_pipe(['submit', '-i'], submitTemplate)
1479
1480             if self.preserveUser:
1481                 if p4User:
1482                     # Get last changelist number. Cannot easily get it from
1483                     # the submit command output as the output is
1484                     # unmarshalled.
1485                     changelist = self.lastP4Changelist()
1486                     self.modifyChangelistUser(changelist, p4User)
1487
1488             # The rename/copy happened by applying a patch that created a
1489             # new file.  This leaves it writable, which confuses p4.
1490             for f in pureRenameCopy:
1491                 p4_sync(f, "-f")
1492
1493         else:
1494             # skip this patch
1495             ret = False
1496             print "Submission cancelled, undoing p4 changes."
1497             for f in editedFiles:
1498                 p4_revert(f)
1499             for f in filesToAdd:
1500                 p4_revert(f)
1501                 os.remove(f)
1502             for f in filesToDelete:
1503                 p4_revert(f)
1504
1505         os.remove(fileName)
1506         return ret
1507
1508     # Export git tags as p4 labels. Create a p4 label and then tag
1509     # with that.
1510     def exportGitTags(self, gitTags):
1511         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1512         if len(validLabelRegexp) == 0:
1513             validLabelRegexp = defaultLabelRegexp
1514         m = re.compile(validLabelRegexp)
1515
1516         for name in gitTags:
1517
1518             if not m.match(name):
1519                 if verbose:
1520                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1521                 continue
1522
1523             # Get the p4 commit this corresponds to
1524             logMessage = extractLogMessageFromGitCommit(name)
1525             values = extractSettingsGitLog(logMessage)
1526
1527             if not values.has_key('change'):
1528                 # a tag pointing to something not sent to p4; ignore
1529                 if verbose:
1530                     print "git tag %s does not give a p4 commit" % name
1531                 continue
1532             else:
1533                 changelist = values['change']
1534
1535             # Get the tag details.
1536             inHeader = True
1537             isAnnotated = False
1538             body = []
1539             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1540                 l = l.strip()
1541                 if inHeader:
1542                     if re.match(r'tag\s+', l):
1543                         isAnnotated = True
1544                     elif re.match(r'\s*$', l):
1545                         inHeader = False
1546                         continue
1547                 else:
1548                     body.append(l)
1549
1550             if not isAnnotated:
1551                 body = ["lightweight tag imported by git p4\n"]
1552
1553             # Create the label - use the same view as the client spec we are using
1554             clientSpec = getClientSpec()
1555
1556             labelTemplate  = "Label: %s\n" % name
1557             labelTemplate += "Description:\n"
1558             for b in body:
1559                 labelTemplate += "\t" + b + "\n"
1560             labelTemplate += "View:\n"
1561             for depot_side in clientSpec.mappings:
1562                 labelTemplate += "\t%s\n" % depot_side
1563
1564             if self.dry_run:
1565                 print "Would create p4 label %s for tag" % name
1566             elif self.prepare_p4_only:
1567                 print "Not creating p4 label %s for tag due to option" \
1568                       " --prepare-p4-only" % name
1569             else:
1570                 p4_write_pipe(["label", "-i"], labelTemplate)
1571
1572                 # Use the label
1573                 p4_system(["tag", "-l", name] +
1574                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1575
1576                 if verbose:
1577                     print "created p4 label for tag %s" % name
1578
1579     def run(self, args):
1580         if len(args) == 0:
1581             self.master = currentGitBranch()
1582             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1583                 die("Detecting current git branch failed!")
1584         elif len(args) == 1:
1585             self.master = args[0]
1586             if not branchExists(self.master):
1587                 die("Branch %s does not exist" % self.master)
1588         else:
1589             return False
1590
1591         allowSubmit = gitConfig("git-p4.allowSubmit")
1592         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1593             die("%s is not in git-p4.allowSubmit" % self.master)
1594
1595         [upstream, settings] = findUpstreamBranchPoint()
1596         self.depotPath = settings['depot-paths'][0]
1597         if len(self.origin) == 0:
1598             self.origin = upstream
1599
1600         if self.preserveUser:
1601             if not self.canChangeChangelists():
1602                 die("Cannot preserve user names without p4 super-user or admin permissions")
1603
1604         # if not set from the command line, try the config file
1605         if self.conflict_behavior is None:
1606             val = gitConfig("git-p4.conflict")
1607             if val:
1608                 if val not in self.conflict_behavior_choices:
1609                     die("Invalid value '%s' for config git-p4.conflict" % val)
1610             else:
1611                 val = "ask"
1612             self.conflict_behavior = val
1613
1614         if self.verbose:
1615             print "Origin branch is " + self.origin
1616
1617         if len(self.depotPath) == 0:
1618             print "Internal error: cannot locate perforce depot path from existing branches"
1619             sys.exit(128)
1620
1621         self.useClientSpec = False
1622         if gitConfigBool("git-p4.useclientspec"):
1623             self.useClientSpec = True
1624         if self.useClientSpec:
1625             self.clientSpecDirs = getClientSpec()
1626
1627         if self.useClientSpec:
1628             # all files are relative to the client spec
1629             self.clientPath = getClientRoot()
1630         else:
1631             self.clientPath = p4Where(self.depotPath)
1632
1633         if self.clientPath == "":
1634             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1635
1636         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1637         self.oldWorkingDirectory = os.getcwd()
1638
1639         # ensure the clientPath exists
1640         new_client_dir = False
1641         if not os.path.exists(self.clientPath):
1642             new_client_dir = True
1643             os.makedirs(self.clientPath)
1644
1645         chdir(self.clientPath, is_client_path=True)
1646         if self.dry_run:
1647             print "Would synchronize p4 checkout in %s" % self.clientPath
1648         else:
1649             print "Synchronizing p4 checkout..."
1650             if new_client_dir:
1651                 # old one was destroyed, and maybe nobody told p4
1652                 p4_sync("...", "-f")
1653             else:
1654                 p4_sync("...")
1655         self.check()
1656
1657         commits = []
1658         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, self.master)]):
1659             commits.append(line.strip())
1660         commits.reverse()
1661
1662         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
1663             self.checkAuthorship = False
1664         else:
1665             self.checkAuthorship = True
1666
1667         if self.preserveUser:
1668             self.checkValidP4Users(commits)
1669
1670         #
1671         # Build up a set of options to be passed to diff when
1672         # submitting each commit to p4.
1673         #
1674         if self.detectRenames:
1675             # command-line -M arg
1676             self.diffOpts = "-M"
1677         else:
1678             # If not explicitly set check the config variable
1679             detectRenames = gitConfig("git-p4.detectRenames")
1680
1681             if detectRenames.lower() == "false" or detectRenames == "":
1682                 self.diffOpts = ""
1683             elif detectRenames.lower() == "true":
1684                 self.diffOpts = "-M"
1685             else:
1686                 self.diffOpts = "-M%s" % detectRenames
1687
1688         # no command-line arg for -C or --find-copies-harder, just
1689         # config variables
1690         detectCopies = gitConfig("git-p4.detectCopies")
1691         if detectCopies.lower() == "false" or detectCopies == "":
1692             pass
1693         elif detectCopies.lower() == "true":
1694             self.diffOpts += " -C"
1695         else:
1696             self.diffOpts += " -C%s" % detectCopies
1697
1698         if gitConfigBool("git-p4.detectCopiesHarder"):
1699             self.diffOpts += " --find-copies-harder"
1700
1701         #
1702         # Apply the commits, one at a time.  On failure, ask if should
1703         # continue to try the rest of the patches, or quit.
1704         #
1705         if self.dry_run:
1706             print "Would apply"
1707         applied = []
1708         last = len(commits) - 1
1709         for i, commit in enumerate(commits):
1710             if self.dry_run:
1711                 print " ", read_pipe(["git", "show", "-s",
1712                                       "--format=format:%h %s", commit])
1713                 ok = True
1714             else:
1715                 ok = self.applyCommit(commit)
1716             if ok:
1717                 applied.append(commit)
1718             else:
1719                 if self.prepare_p4_only and i < last:
1720                     print "Processing only the first commit due to option" \
1721                           " --prepare-p4-only"
1722                     break
1723                 if i < last:
1724                     quit = False
1725                     while True:
1726                         # prompt for what to do, or use the option/variable
1727                         if self.conflict_behavior == "ask":
1728                             print "What do you want to do?"
1729                             response = raw_input("[s]kip this commit but apply"
1730                                                  " the rest, or [q]uit? ")
1731                             if not response:
1732                                 continue
1733                         elif self.conflict_behavior == "skip":
1734                             response = "s"
1735                         elif self.conflict_behavior == "quit":
1736                             response = "q"
1737                         else:
1738                             die("Unknown conflict_behavior '%s'" %
1739                                 self.conflict_behavior)
1740
1741                         if response[0] == "s":
1742                             print "Skipping this commit, but applying the rest"
1743                             break
1744                         if response[0] == "q":
1745                             print "Quitting"
1746                             quit = True
1747                             break
1748                     if quit:
1749                         break
1750
1751         chdir(self.oldWorkingDirectory)
1752
1753         if self.dry_run:
1754             pass
1755         elif self.prepare_p4_only:
1756             pass
1757         elif len(commits) == len(applied):
1758             print "All commits applied!"
1759
1760             sync = P4Sync()
1761             if self.branch:
1762                 sync.branch = self.branch
1763             sync.run([])
1764
1765             rebase = P4Rebase()
1766             rebase.rebase()
1767
1768         else:
1769             if len(applied) == 0:
1770                 print "No commits applied."
1771             else:
1772                 print "Applied only the commits marked with '*':"
1773                 for c in commits:
1774                     if c in applied:
1775                         star = "*"
1776                     else:
1777                         star = " "
1778                     print star, read_pipe(["git", "show", "-s",
1779                                            "--format=format:%h %s",  c])
1780                 print "You will have to do 'git p4 sync' and rebase."
1781
1782         if gitConfigBool("git-p4.exportLabels"):
1783             self.exportLabels = True
1784
1785         if self.exportLabels:
1786             p4Labels = getP4Labels(self.depotPath)
1787             gitTags = getGitTags()
1788
1789             missingGitTags = gitTags - p4Labels
1790             self.exportGitTags(missingGitTags)
1791
1792         # exit with error unless everything applied perfectly
1793         if len(commits) != len(applied):
1794                 sys.exit(1)
1795
1796         return True
1797
1798 class View(object):
1799     """Represent a p4 view ("p4 help views"), and map files in a
1800        repo according to the view."""
1801
1802     def __init__(self, client_name):
1803         self.mappings = []
1804         self.client_prefix = "//%s/" % client_name
1805         # cache results of "p4 where" to lookup client file locations
1806         self.client_spec_path_cache = {}
1807
1808     def append(self, view_line):
1809         """Parse a view line, splitting it into depot and client
1810            sides.  Append to self.mappings, preserving order.  This
1811            is only needed for tag creation."""
1812
1813         # Split the view line into exactly two words.  P4 enforces
1814         # structure on these lines that simplifies this quite a bit.
1815         #
1816         # Either or both words may be double-quoted.
1817         # Single quotes do not matter.
1818         # Double-quote marks cannot occur inside the words.
1819         # A + or - prefix is also inside the quotes.
1820         # There are no quotes unless they contain a space.
1821         # The line is already white-space stripped.
1822         # The two words are separated by a single space.
1823         #
1824         if view_line[0] == '"':
1825             # First word is double quoted.  Find its end.
1826             close_quote_index = view_line.find('"', 1)
1827             if close_quote_index <= 0:
1828                 die("No first-word closing quote found: %s" % view_line)
1829             depot_side = view_line[1:close_quote_index]
1830             # skip closing quote and space
1831             rhs_index = close_quote_index + 1 + 1
1832         else:
1833             space_index = view_line.find(" ")
1834             if space_index <= 0:
1835                 die("No word-splitting space found: %s" % view_line)
1836             depot_side = view_line[0:space_index]
1837             rhs_index = space_index + 1
1838
1839         # prefix + means overlay on previous mapping
1840         if depot_side.startswith("+"):
1841             depot_side = depot_side[1:]
1842
1843         # prefix - means exclude this path, leave out of mappings
1844         exclude = False
1845         if depot_side.startswith("-"):
1846             exclude = True
1847             depot_side = depot_side[1:]
1848
1849         if not exclude:
1850             self.mappings.append(depot_side)
1851
1852     def convert_client_path(self, clientFile):
1853         # chop off //client/ part to make it relative
1854         if not clientFile.startswith(self.client_prefix):
1855             die("No prefix '%s' on clientFile '%s'" %
1856                 (self.client_prefix, clientFile))
1857         return clientFile[len(self.client_prefix):]
1858
1859     def update_client_spec_path_cache(self, files):
1860         """ Caching file paths by "p4 where" batch query """
1861
1862         # List depot file paths exclude that already cached
1863         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
1864
1865         if len(fileArgs) == 0:
1866             return  # All files in cache
1867
1868         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
1869         for res in where_result:
1870             if "code" in res and res["code"] == "error":
1871                 # assume error is "... file(s) not in client view"
1872                 continue
1873             if "clientFile" not in res:
1874                 die("No clientFile in 'p4 where' output")
1875             if "unmap" in res:
1876                 # it will list all of them, but only one not unmap-ped
1877                 continue
1878             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
1879
1880         # not found files or unmap files set to ""
1881         for depotFile in fileArgs:
1882             if depotFile not in self.client_spec_path_cache:
1883                 self.client_spec_path_cache[depotFile] = ""
1884
1885     def map_in_client(self, depot_path):
1886         """Return the relative location in the client where this
1887            depot file should live.  Returns "" if the file should
1888            not be mapped in the client."""
1889
1890         if depot_path in self.client_spec_path_cache:
1891             return self.client_spec_path_cache[depot_path]
1892
1893         die( "Error: %s is not found in client spec path" % depot_path )
1894         return ""
1895
1896 class P4Sync(Command, P4UserMap):
1897     delete_actions = ( "delete", "move/delete", "purge" )
1898
1899     def __init__(self):
1900         Command.__init__(self)
1901         P4UserMap.__init__(self)
1902         self.options = [
1903                 optparse.make_option("--branch", dest="branch"),
1904                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1905                 optparse.make_option("--changesfile", dest="changesFile"),
1906                 optparse.make_option("--silent", dest="silent", action="store_true"),
1907                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1908                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
1909                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1910                                      help="Import into refs/heads/ , not refs/remotes"),
1911                 optparse.make_option("--max-changes", dest="maxChanges"),
1912                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1913                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1914                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1915                                      help="Only sync files that are included in the Perforce Client Spec")
1916         ]
1917         self.description = """Imports from Perforce into a git repository.\n
1918     example:
1919     //depot/my/project/ -- to import the current head
1920     //depot/my/project/@all -- to import everything
1921     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1922
1923     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1924
1925         self.usage += " //depot/path[@revRange]"
1926         self.silent = False
1927         self.createdBranches = set()
1928         self.committedChanges = set()
1929         self.branch = ""
1930         self.detectBranches = False
1931         self.detectLabels = False
1932         self.importLabels = False
1933         self.changesFile = ""
1934         self.syncWithOrigin = True
1935         self.importIntoRemotes = True
1936         self.maxChanges = ""
1937         self.keepRepoPath = False
1938         self.depotPaths = None
1939         self.p4BranchesInGit = []
1940         self.cloneExclude = []
1941         self.useClientSpec = False
1942         self.useClientSpec_from_options = False
1943         self.clientSpecDirs = None
1944         self.tempBranches = []
1945         self.tempBranchLocation = "git-p4-tmp"
1946
1947         if gitConfig("git-p4.syncFromOrigin") == "false":
1948             self.syncWithOrigin = False
1949
1950     # Force a checkpoint in fast-import and wait for it to finish
1951     def checkpoint(self):
1952         self.gitStream.write("checkpoint\n\n")
1953         self.gitStream.write("progress checkpoint\n\n")
1954         out = self.gitOutput.readline()
1955         if self.verbose:
1956             print "checkpoint finished: " + out
1957
1958     def extractFilesFromCommit(self, commit):
1959         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1960                              for path in self.cloneExclude]
1961         files = []
1962         fnum = 0
1963         while commit.has_key("depotFile%s" % fnum):
1964             path =  commit["depotFile%s" % fnum]
1965
1966             if [p for p in self.cloneExclude
1967                 if p4PathStartsWith(path, p)]:
1968                 found = False
1969             else:
1970                 found = [p for p in self.depotPaths
1971                          if p4PathStartsWith(path, p)]
1972             if not found:
1973                 fnum = fnum + 1
1974                 continue
1975
1976             file = {}
1977             file["path"] = path
1978             file["rev"] = commit["rev%s" % fnum]
1979             file["action"] = commit["action%s" % fnum]
1980             file["type"] = commit["type%s" % fnum]
1981             files.append(file)
1982             fnum = fnum + 1
1983         return files
1984
1985     def stripRepoPath(self, path, prefixes):
1986         """When streaming files, this is called to map a p4 depot path
1987            to where it should go in git.  The prefixes are either
1988            self.depotPaths, or self.branchPrefixes in the case of
1989            branch detection."""
1990
1991         if self.useClientSpec:
1992             # branch detection moves files up a level (the branch name)
1993             # from what client spec interpretation gives
1994             path = self.clientSpecDirs.map_in_client(path)
1995             if self.detectBranches:
1996                 for b in self.knownBranches:
1997                     if path.startswith(b + "/"):
1998                         path = path[len(b)+1:]
1999
2000         elif self.keepRepoPath:
2001             # Preserve everything in relative path name except leading
2002             # //depot/; just look at first prefix as they all should
2003             # be in the same depot.
2004             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2005             if p4PathStartsWith(path, depot):
2006                 path = path[len(depot):]
2007
2008         else:
2009             for p in prefixes:
2010                 if p4PathStartsWith(path, p):
2011                     path = path[len(p):]
2012                     break
2013
2014         path = wildcard_decode(path)
2015         return path
2016
2017     def splitFilesIntoBranches(self, commit):
2018         """Look at each depotFile in the commit to figure out to what
2019            branch it belongs."""
2020
2021         if self.clientSpecDirs:
2022             files = self.extractFilesFromCommit(commit)
2023             self.clientSpecDirs.update_client_spec_path_cache(files)
2024
2025         branches = {}
2026         fnum = 0
2027         while commit.has_key("depotFile%s" % fnum):
2028             path =  commit["depotFile%s" % fnum]
2029             found = [p for p in self.depotPaths
2030                      if p4PathStartsWith(path, p)]
2031             if not found:
2032                 fnum = fnum + 1
2033                 continue
2034
2035             file = {}
2036             file["path"] = path
2037             file["rev"] = commit["rev%s" % fnum]
2038             file["action"] = commit["action%s" % fnum]
2039             file["type"] = commit["type%s" % fnum]
2040             fnum = fnum + 1
2041
2042             # start with the full relative path where this file would
2043             # go in a p4 client
2044             if self.useClientSpec:
2045                 relPath = self.clientSpecDirs.map_in_client(path)
2046             else:
2047                 relPath = self.stripRepoPath(path, self.depotPaths)
2048
2049             for branch in self.knownBranches.keys():
2050                 # add a trailing slash so that a commit into qt/4.2foo
2051                 # doesn't end up in qt/4.2, e.g.
2052                 if relPath.startswith(branch + "/"):
2053                     if branch not in branches:
2054                         branches[branch] = []
2055                     branches[branch].append(file)
2056                     break
2057
2058         return branches
2059
2060     # output one file from the P4 stream
2061     # - helper for streamP4Files
2062
2063     def streamOneP4File(self, file, contents):
2064         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2065         if verbose:
2066             sys.stderr.write("%s\n" % relPath)
2067
2068         (type_base, type_mods) = split_p4_type(file["type"])
2069
2070         git_mode = "100644"
2071         if "x" in type_mods:
2072             git_mode = "100755"
2073         if type_base == "symlink":
2074             git_mode = "120000"
2075             # p4 print on a symlink sometimes contains "target\n";
2076             # if it does, remove the newline
2077             data = ''.join(contents)
2078             if not data:
2079                 # Some version of p4 allowed creating a symlink that pointed
2080                 # to nothing.  This causes p4 errors when checking out such
2081                 # a change, and errors here too.  Work around it by ignoring
2082                 # the bad symlink; hopefully a future change fixes it.
2083                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2084                 return
2085             elif data[-1] == '\n':
2086                 contents = [data[:-1]]
2087             else:
2088                 contents = [data]
2089
2090         if type_base == "utf16":
2091             # p4 delivers different text in the python output to -G
2092             # than it does when using "print -o", or normal p4 client
2093             # operations.  utf16 is converted to ascii or utf8, perhaps.
2094             # But ascii text saved as -t utf16 is completely mangled.
2095             # Invoke print -o to get the real contents.
2096             #
2097             # On windows, the newlines will always be mangled by print, so put
2098             # them back too.  This is not needed to the cygwin windows version,
2099             # just the native "NT" type.
2100             #
2101             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
2102             if p4_version_string().find("/NT") >= 0:
2103                 text = text.replace("\r\n", "\n")
2104             contents = [ text ]
2105
2106         if type_base == "apple":
2107             # Apple filetype files will be streamed as a concatenation of
2108             # its appledouble header and the contents.  This is useless
2109             # on both macs and non-macs.  If using "print -q -o xx", it
2110             # will create "xx" with the data, and "%xx" with the header.
2111             # This is also not very useful.
2112             #
2113             # Ideally, someday, this script can learn how to generate
2114             # appledouble files directly and import those to git, but
2115             # non-mac machines can never find a use for apple filetype.
2116             print "\nIgnoring apple filetype file %s" % file['depotFile']
2117             return
2118
2119         # Note that we do not try to de-mangle keywords on utf16 files,
2120         # even though in theory somebody may want that.
2121         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2122         if pattern:
2123             regexp = re.compile(pattern, re.VERBOSE)
2124             text = ''.join(contents)
2125             text = regexp.sub(r'$\1$', text)
2126             contents = [ text ]
2127
2128         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
2129
2130         # total length...
2131         length = 0
2132         for d in contents:
2133             length = length + len(d)
2134
2135         self.gitStream.write("data %d\n" % length)
2136         for d in contents:
2137             self.gitStream.write(d)
2138         self.gitStream.write("\n")
2139
2140     def streamOneP4Deletion(self, file):
2141         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2142         if verbose:
2143             sys.stderr.write("delete %s\n" % relPath)
2144         self.gitStream.write("D %s\n" % relPath)
2145
2146     # handle another chunk of streaming data
2147     def streamP4FilesCb(self, marshalled):
2148
2149         # catch p4 errors and complain
2150         err = None
2151         if "code" in marshalled:
2152             if marshalled["code"] == "error":
2153                 if "data" in marshalled:
2154                     err = marshalled["data"].rstrip()
2155         if err:
2156             f = None
2157             if self.stream_have_file_info:
2158                 if "depotFile" in self.stream_file:
2159                     f = self.stream_file["depotFile"]
2160             # force a failure in fast-import, else an empty
2161             # commit will be made
2162             self.gitStream.write("\n")
2163             self.gitStream.write("die-now\n")
2164             self.gitStream.close()
2165             # ignore errors, but make sure it exits first
2166             self.importProcess.wait()
2167             if f:
2168                 die("Error from p4 print for %s: %s" % (f, err))
2169             else:
2170                 die("Error from p4 print: %s" % err)
2171
2172         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2173             # start of a new file - output the old one first
2174             self.streamOneP4File(self.stream_file, self.stream_contents)
2175             self.stream_file = {}
2176             self.stream_contents = []
2177             self.stream_have_file_info = False
2178
2179         # pick up the new file information... for the
2180         # 'data' field we need to append to our array
2181         for k in marshalled.keys():
2182             if k == 'data':
2183                 self.stream_contents.append(marshalled['data'])
2184             else:
2185                 self.stream_file[k] = marshalled[k]
2186
2187         self.stream_have_file_info = True
2188
2189     # Stream directly from "p4 files" into "git fast-import"
2190     def streamP4Files(self, files):
2191         filesForCommit = []
2192         filesToRead = []
2193         filesToDelete = []
2194
2195         for f in files:
2196             # if using a client spec, only add the files that have
2197             # a path in the client
2198             if self.clientSpecDirs:
2199                 if self.clientSpecDirs.map_in_client(f['path']) == "":
2200                     continue
2201
2202             filesForCommit.append(f)
2203             if f['action'] in self.delete_actions:
2204                 filesToDelete.append(f)
2205             else:
2206                 filesToRead.append(f)
2207
2208         # deleted files...
2209         for f in filesToDelete:
2210             self.streamOneP4Deletion(f)
2211
2212         if len(filesToRead) > 0:
2213             self.stream_file = {}
2214             self.stream_contents = []
2215             self.stream_have_file_info = False
2216
2217             # curry self argument
2218             def streamP4FilesCbSelf(entry):
2219                 self.streamP4FilesCb(entry)
2220
2221             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2222
2223             p4CmdList(["-x", "-", "print"],
2224                       stdin=fileArgs,
2225                       cb=streamP4FilesCbSelf)
2226
2227             # do the last chunk
2228             if self.stream_file.has_key('depotFile'):
2229                 self.streamOneP4File(self.stream_file, self.stream_contents)
2230
2231     def make_email(self, userid):
2232         if userid in self.users:
2233             return self.users[userid]
2234         else:
2235             return "%s <a@b>" % userid
2236
2237     # Stream a p4 tag
2238     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2239         if verbose:
2240             print "writing tag %s for commit %s" % (labelName, commit)
2241         gitStream.write("tag %s\n" % labelName)
2242         gitStream.write("from %s\n" % commit)
2243
2244         if labelDetails.has_key('Owner'):
2245             owner = labelDetails["Owner"]
2246         else:
2247             owner = None
2248
2249         # Try to use the owner of the p4 label, or failing that,
2250         # the current p4 user id.
2251         if owner:
2252             email = self.make_email(owner)
2253         else:
2254             email = self.make_email(self.p4UserId())
2255         tagger = "%s %s %s" % (email, epoch, self.tz)
2256
2257         gitStream.write("tagger %s\n" % tagger)
2258
2259         print "labelDetails=",labelDetails
2260         if labelDetails.has_key('Description'):
2261             description = labelDetails['Description']
2262         else:
2263             description = 'Label from git p4'
2264
2265         gitStream.write("data %d\n" % len(description))
2266         gitStream.write(description)
2267         gitStream.write("\n")
2268
2269     def commit(self, details, files, branch, parent = ""):
2270         epoch = details["time"]
2271         author = details["user"]
2272
2273         if self.verbose:
2274             print "commit into %s" % branch
2275
2276         # start with reading files; if that fails, we should not
2277         # create a commit.
2278         new_files = []
2279         for f in files:
2280             if [p for p in self.branchPrefixes if p4PathStartsWith(f['path'], p)]:
2281                 new_files.append (f)
2282             else:
2283                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
2284
2285         if self.clientSpecDirs:
2286             self.clientSpecDirs.update_client_spec_path_cache(files)
2287
2288         self.gitStream.write("commit %s\n" % branch)
2289 #        gitStream.write("mark :%s\n" % details["change"])
2290         self.committedChanges.add(int(details["change"]))
2291         committer = ""
2292         if author not in self.users:
2293             self.getUserMapFromPerforceServer()
2294         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2295
2296         self.gitStream.write("committer %s\n" % committer)
2297
2298         self.gitStream.write("data <<EOT\n")
2299         self.gitStream.write(details["desc"])
2300         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2301                              (','.join(self.branchPrefixes), details["change"]))
2302         if len(details['options']) > 0:
2303             self.gitStream.write(": options = %s" % details['options'])
2304         self.gitStream.write("]\nEOT\n\n")
2305
2306         if len(parent) > 0:
2307             if self.verbose:
2308                 print "parent %s" % parent
2309             self.gitStream.write("from %s\n" % parent)
2310
2311         self.streamP4Files(new_files)
2312         self.gitStream.write("\n")
2313
2314         change = int(details["change"])
2315
2316         if self.labels.has_key(change):
2317             label = self.labels[change]
2318             labelDetails = label[0]
2319             labelRevisions = label[1]
2320             if self.verbose:
2321                 print "Change %s is labelled %s" % (change, labelDetails)
2322
2323             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2324                                                 for p in self.branchPrefixes])
2325
2326             if len(files) == len(labelRevisions):
2327
2328                 cleanedFiles = {}
2329                 for info in files:
2330                     if info["action"] in self.delete_actions:
2331                         continue
2332                     cleanedFiles[info["depotFile"]] = info["rev"]
2333
2334                 if cleanedFiles == labelRevisions:
2335                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2336
2337                 else:
2338                     if not self.silent:
2339                         print ("Tag %s does not match with change %s: files do not match."
2340                                % (labelDetails["label"], change))
2341
2342             else:
2343                 if not self.silent:
2344                     print ("Tag %s does not match with change %s: file count is different."
2345                            % (labelDetails["label"], change))
2346
2347     # Build a dictionary of changelists and labels, for "detect-labels" option.
2348     def getLabels(self):
2349         self.labels = {}
2350
2351         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2352         if len(l) > 0 and not self.silent:
2353             print "Finding files belonging to labels in %s" % `self.depotPaths`
2354
2355         for output in l:
2356             label = output["label"]
2357             revisions = {}
2358             newestChange = 0
2359             if self.verbose:
2360                 print "Querying files for label %s" % label
2361             for file in p4CmdList(["files"] +
2362                                       ["%s...@%s" % (p, label)
2363                                           for p in self.depotPaths]):
2364                 revisions[file["depotFile"]] = file["rev"]
2365                 change = int(file["change"])
2366                 if change > newestChange:
2367                     newestChange = change
2368
2369             self.labels[newestChange] = [output, revisions]
2370
2371         if self.verbose:
2372             print "Label changes: %s" % self.labels.keys()
2373
2374     # Import p4 labels as git tags. A direct mapping does not
2375     # exist, so assume that if all the files are at the same revision
2376     # then we can use that, or it's something more complicated we should
2377     # just ignore.
2378     def importP4Labels(self, stream, p4Labels):
2379         if verbose:
2380             print "import p4 labels: " + ' '.join(p4Labels)
2381
2382         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2383         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2384         if len(validLabelRegexp) == 0:
2385             validLabelRegexp = defaultLabelRegexp
2386         m = re.compile(validLabelRegexp)
2387
2388         for name in p4Labels:
2389             commitFound = False
2390
2391             if not m.match(name):
2392                 if verbose:
2393                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2394                 continue
2395
2396             if name in ignoredP4Labels:
2397                 continue
2398
2399             labelDetails = p4CmdList(['label', "-o", name])[0]
2400
2401             # get the most recent changelist for each file in this label
2402             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2403                                 for p in self.depotPaths])
2404
2405             if change.has_key('change'):
2406                 # find the corresponding git commit; take the oldest commit
2407                 changelist = int(change['change'])
2408                 gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2409                      "--reverse", ":/\[git-p4:.*change = %d\]" % changelist])
2410                 if len(gitCommit) == 0:
2411                     print "could not find git commit for changelist %d" % changelist
2412                 else:
2413                     gitCommit = gitCommit.strip()
2414                     commitFound = True
2415                     # Convert from p4 time format
2416                     try:
2417                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2418                     except ValueError:
2419                         print "Could not convert label time %s" % labelDetails['Update']
2420                         tmwhen = 1
2421
2422                     when = int(time.mktime(tmwhen))
2423                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2424                     if verbose:
2425                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2426             else:
2427                 if verbose:
2428                     print "Label %s has no changelists - possibly deleted?" % name
2429
2430             if not commitFound:
2431                 # We can't import this label; don't try again as it will get very
2432                 # expensive repeatedly fetching all the files for labels that will
2433                 # never be imported. If the label is moved in the future, the
2434                 # ignore will need to be removed manually.
2435                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2436
2437     def guessProjectName(self):
2438         for p in self.depotPaths:
2439             if p.endswith("/"):
2440                 p = p[:-1]
2441             p = p[p.strip().rfind("/") + 1:]
2442             if not p.endswith("/"):
2443                p += "/"
2444             return p
2445
2446     def getBranchMapping(self):
2447         lostAndFoundBranches = set()
2448
2449         user = gitConfig("git-p4.branchUser")
2450         if len(user) > 0:
2451             command = "branches -u %s" % user
2452         else:
2453             command = "branches"
2454
2455         for info in p4CmdList(command):
2456             details = p4Cmd(["branch", "-o", info["branch"]])
2457             viewIdx = 0
2458             while details.has_key("View%s" % viewIdx):
2459                 paths = details["View%s" % viewIdx].split(" ")
2460                 viewIdx = viewIdx + 1
2461                 # require standard //depot/foo/... //depot/bar/... mapping
2462                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2463                     continue
2464                 source = paths[0]
2465                 destination = paths[1]
2466                 ## HACK
2467                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2468                     source = source[len(self.depotPaths[0]):-4]
2469                     destination = destination[len(self.depotPaths[0]):-4]
2470
2471                     if destination in self.knownBranches:
2472                         if not self.silent:
2473                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2474                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2475                         continue
2476
2477                     self.knownBranches[destination] = source
2478
2479                     lostAndFoundBranches.discard(destination)
2480
2481                     if source not in self.knownBranches:
2482                         lostAndFoundBranches.add(source)
2483
2484         # Perforce does not strictly require branches to be defined, so we also
2485         # check git config for a branch list.
2486         #
2487         # Example of branch definition in git config file:
2488         # [git-p4]
2489         #   branchList=main:branchA
2490         #   branchList=main:branchB
2491         #   branchList=branchA:branchC
2492         configBranches = gitConfigList("git-p4.branchList")
2493         for branch in configBranches:
2494             if branch:
2495                 (source, destination) = branch.split(":")
2496                 self.knownBranches[destination] = source
2497
2498                 lostAndFoundBranches.discard(destination)
2499
2500                 if source not in self.knownBranches:
2501                     lostAndFoundBranches.add(source)
2502
2503
2504         for branch in lostAndFoundBranches:
2505             self.knownBranches[branch] = branch
2506
2507     def getBranchMappingFromGitBranches(self):
2508         branches = p4BranchesInGit(self.importIntoRemotes)
2509         for branch in branches.keys():
2510             if branch == "master":
2511                 branch = "main"
2512             else:
2513                 branch = branch[len(self.projectName):]
2514             self.knownBranches[branch] = branch
2515
2516     def updateOptionDict(self, d):
2517         option_keys = {}
2518         if self.keepRepoPath:
2519             option_keys['keepRepoPath'] = 1
2520
2521         d["options"] = ' '.join(sorted(option_keys.keys()))
2522
2523     def readOptions(self, d):
2524         self.keepRepoPath = (d.has_key('options')
2525                              and ('keepRepoPath' in d['options']))
2526
2527     def gitRefForBranch(self, branch):
2528         if branch == "main":
2529             return self.refPrefix + "master"
2530
2531         if len(branch) <= 0:
2532             return branch
2533
2534         return self.refPrefix + self.projectName + branch
2535
2536     def gitCommitByP4Change(self, ref, change):
2537         if self.verbose:
2538             print "looking in ref " + ref + " for change %s using bisect..." % change
2539
2540         earliestCommit = ""
2541         latestCommit = parseRevision(ref)
2542
2543         while True:
2544             if self.verbose:
2545                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2546             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2547             if len(next) == 0:
2548                 if self.verbose:
2549                     print "argh"
2550                 return ""
2551             log = extractLogMessageFromGitCommit(next)
2552             settings = extractSettingsGitLog(log)
2553             currentChange = int(settings['change'])
2554             if self.verbose:
2555                 print "current change %s" % currentChange
2556
2557             if currentChange == change:
2558                 if self.verbose:
2559                     print "found %s" % next
2560                 return next
2561
2562             if currentChange < change:
2563                 earliestCommit = "^%s" % next
2564             else:
2565                 latestCommit = "%s" % next
2566
2567         return ""
2568
2569     def importNewBranch(self, branch, maxChange):
2570         # make fast-import flush all changes to disk and update the refs using the checkpoint
2571         # command so that we can try to find the branch parent in the git history
2572         self.gitStream.write("checkpoint\n\n");
2573         self.gitStream.flush();
2574         branchPrefix = self.depotPaths[0] + branch + "/"
2575         range = "@1,%s" % maxChange
2576         #print "prefix" + branchPrefix
2577         changes = p4ChangesForPaths([branchPrefix], range)
2578         if len(changes) <= 0:
2579             return False
2580         firstChange = changes[0]
2581         #print "first change in branch: %s" % firstChange
2582         sourceBranch = self.knownBranches[branch]
2583         sourceDepotPath = self.depotPaths[0] + sourceBranch
2584         sourceRef = self.gitRefForBranch(sourceBranch)
2585         #print "source " + sourceBranch
2586
2587         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
2588         #print "branch parent: %s" % branchParentChange
2589         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
2590         if len(gitParent) > 0:
2591             self.initialParents[self.gitRefForBranch(branch)] = gitParent
2592             #print "parent git commit: %s" % gitParent
2593
2594         self.importChanges(changes)
2595         return True
2596
2597     def searchParent(self, parent, branch, target):
2598         parentFound = False
2599         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
2600                                      "--no-merges", parent]):
2601             blob = blob.strip()
2602             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
2603                 parentFound = True
2604                 if self.verbose:
2605                     print "Found parent of %s in commit %s" % (branch, blob)
2606                 break
2607         if parentFound:
2608             return blob
2609         else:
2610             return None
2611
2612     def importChanges(self, changes):
2613         cnt = 1
2614         for change in changes:
2615             description = p4_describe(change)
2616             self.updateOptionDict(description)
2617
2618             if not self.silent:
2619                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
2620                 sys.stdout.flush()
2621             cnt = cnt + 1
2622
2623             try:
2624                 if self.detectBranches:
2625                     branches = self.splitFilesIntoBranches(description)
2626                     for branch in branches.keys():
2627                         ## HACK  --hwn
2628                         branchPrefix = self.depotPaths[0] + branch + "/"
2629                         self.branchPrefixes = [ branchPrefix ]
2630
2631                         parent = ""
2632
2633                         filesForCommit = branches[branch]
2634
2635                         if self.verbose:
2636                             print "branch is %s" % branch
2637
2638                         self.updatedBranches.add(branch)
2639
2640                         if branch not in self.createdBranches:
2641                             self.createdBranches.add(branch)
2642                             parent = self.knownBranches[branch]
2643                             if parent == branch:
2644                                 parent = ""
2645                             else:
2646                                 fullBranch = self.projectName + branch
2647                                 if fullBranch not in self.p4BranchesInGit:
2648                                     if not self.silent:
2649                                         print("\n    Importing new branch %s" % fullBranch);
2650                                     if self.importNewBranch(branch, change - 1):
2651                                         parent = ""
2652                                         self.p4BranchesInGit.append(fullBranch)
2653                                     if not self.silent:
2654                                         print("\n    Resuming with change %s" % change);
2655
2656                                 if self.verbose:
2657                                     print "parent determined through known branches: %s" % parent
2658
2659                         branch = self.gitRefForBranch(branch)
2660                         parent = self.gitRefForBranch(parent)
2661
2662                         if self.verbose:
2663                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
2664
2665                         if len(parent) == 0 and branch in self.initialParents:
2666                             parent = self.initialParents[branch]
2667                             del self.initialParents[branch]
2668
2669                         blob = None
2670                         if len(parent) > 0:
2671                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
2672                             if self.verbose:
2673                                 print "Creating temporary branch: " + tempBranch
2674                             self.commit(description, filesForCommit, tempBranch)
2675                             self.tempBranches.append(tempBranch)
2676                             self.checkpoint()
2677                             blob = self.searchParent(parent, branch, tempBranch)
2678                         if blob:
2679                             self.commit(description, filesForCommit, branch, blob)
2680                         else:
2681                             if self.verbose:
2682                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
2683                             self.commit(description, filesForCommit, branch, parent)
2684                 else:
2685                     files = self.extractFilesFromCommit(description)
2686                     self.commit(description, files, self.branch,
2687                                 self.initialParent)
2688                     # only needed once, to connect to the previous commit
2689                     self.initialParent = ""
2690             except IOError:
2691                 print self.gitError.read()
2692                 sys.exit(1)
2693
2694     def importHeadRevision(self, revision):
2695         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
2696
2697         details = {}
2698         details["user"] = "git perforce import user"
2699         details["desc"] = ("Initial import of %s from the state at revision %s\n"
2700                            % (' '.join(self.depotPaths), revision))
2701         details["change"] = revision
2702         newestRevision = 0
2703
2704         fileCnt = 0
2705         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
2706
2707         for info in p4CmdList(["files"] + fileArgs):
2708
2709             if 'code' in info and info['code'] == 'error':
2710                 sys.stderr.write("p4 returned an error: %s\n"
2711                                  % info['data'])
2712                 if info['data'].find("must refer to client") >= 0:
2713                     sys.stderr.write("This particular p4 error is misleading.\n")
2714                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
2715                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
2716                 sys.exit(1)
2717             if 'p4ExitCode' in info:
2718                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
2719                 sys.exit(1)
2720
2721
2722             change = int(info["change"])
2723             if change > newestRevision:
2724                 newestRevision = change
2725
2726             if info["action"] in self.delete_actions:
2727                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
2728                 #fileCnt = fileCnt + 1
2729                 continue
2730
2731             for prop in ["depotFile", "rev", "action", "type" ]:
2732                 details["%s%s" % (prop, fileCnt)] = info[prop]
2733
2734             fileCnt = fileCnt + 1
2735
2736         details["change"] = newestRevision
2737
2738         # Use time from top-most change so that all git p4 clones of
2739         # the same p4 repo have the same commit SHA1s.
2740         res = p4_describe(newestRevision)
2741         details["time"] = res["time"]
2742
2743         self.updateOptionDict(details)
2744         try:
2745             self.commit(details, self.extractFilesFromCommit(details), self.branch)
2746         except IOError:
2747             print "IO error with git fast-import. Is your git version recent enough?"
2748             print self.gitError.read()
2749
2750
2751     def run(self, args):
2752         self.depotPaths = []
2753         self.changeRange = ""
2754         self.previousDepotPaths = []
2755         self.hasOrigin = False
2756
2757         # map from branch depot path to parent branch
2758         self.knownBranches = {}
2759         self.initialParents = {}
2760
2761         if self.importIntoRemotes:
2762             self.refPrefix = "refs/remotes/p4/"
2763         else:
2764             self.refPrefix = "refs/heads/p4/"
2765
2766         if self.syncWithOrigin:
2767             self.hasOrigin = originP4BranchesExist()
2768             if self.hasOrigin:
2769                 if not self.silent:
2770                     print 'Syncing with origin first, using "git fetch origin"'
2771                 system("git fetch origin")
2772
2773         branch_arg_given = bool(self.branch)
2774         if len(self.branch) == 0:
2775             self.branch = self.refPrefix + "master"
2776             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
2777                 system("git update-ref %s refs/heads/p4" % self.branch)
2778                 system("git branch -D p4")
2779
2780         # accept either the command-line option, or the configuration variable
2781         if self.useClientSpec:
2782             # will use this after clone to set the variable
2783             self.useClientSpec_from_options = True
2784         else:
2785             if gitConfigBool("git-p4.useclientspec"):
2786                 self.useClientSpec = True
2787         if self.useClientSpec:
2788             self.clientSpecDirs = getClientSpec()
2789
2790         # TODO: should always look at previous commits,
2791         # merge with previous imports, if possible.
2792         if args == []:
2793             if self.hasOrigin:
2794                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
2795
2796             # branches holds mapping from branch name to sha1
2797             branches = p4BranchesInGit(self.importIntoRemotes)
2798
2799             # restrict to just this one, disabling detect-branches
2800             if branch_arg_given:
2801                 short = self.branch.split("/")[-1]
2802                 if short in branches:
2803                     self.p4BranchesInGit = [ short ]
2804             else:
2805                 self.p4BranchesInGit = branches.keys()
2806
2807             if len(self.p4BranchesInGit) > 1:
2808                 if not self.silent:
2809                     print "Importing from/into multiple branches"
2810                 self.detectBranches = True
2811                 for branch in branches.keys():
2812                     self.initialParents[self.refPrefix + branch] = \
2813                         branches[branch]
2814
2815             if self.verbose:
2816                 print "branches: %s" % self.p4BranchesInGit
2817
2818             p4Change = 0
2819             for branch in self.p4BranchesInGit:
2820                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2821
2822                 settings = extractSettingsGitLog(logMsg)
2823
2824                 self.readOptions(settings)
2825                 if (settings.has_key('depot-paths')
2826                     and settings.has_key ('change')):
2827                     change = int(settings['change']) + 1
2828                     p4Change = max(p4Change, change)
2829
2830                     depotPaths = sorted(settings['depot-paths'])
2831                     if self.previousDepotPaths == []:
2832                         self.previousDepotPaths = depotPaths
2833                     else:
2834                         paths = []
2835                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2836                             prev_list = prev.split("/")
2837                             cur_list = cur.split("/")
2838                             for i in range(0, min(len(cur_list), len(prev_list))):
2839                                 if cur_list[i] <> prev_list[i]:
2840                                     i = i - 1
2841                                     break
2842
2843                             paths.append ("/".join(cur_list[:i + 1]))
2844
2845                         self.previousDepotPaths = paths
2846
2847             if p4Change > 0:
2848                 self.depotPaths = sorted(self.previousDepotPaths)
2849                 self.changeRange = "@%s,#head" % p4Change
2850                 if not self.silent and not self.detectBranches:
2851                     print "Performing incremental import into %s git branch" % self.branch
2852
2853         # accept multiple ref name abbreviations:
2854         #    refs/foo/bar/branch -> use it exactly
2855         #    p4/branch -> prepend refs/remotes/ or refs/heads/
2856         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
2857         if not self.branch.startswith("refs/"):
2858             if self.importIntoRemotes:
2859                 prepend = "refs/remotes/"
2860             else:
2861                 prepend = "refs/heads/"
2862             if not self.branch.startswith("p4/"):
2863                 prepend += "p4/"
2864             self.branch = prepend + self.branch
2865
2866         if len(args) == 0 and self.depotPaths:
2867             if not self.silent:
2868                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2869         else:
2870             if self.depotPaths and self.depotPaths != args:
2871                 print ("previous import used depot path %s and now %s was specified. "
2872                        "This doesn't work!" % (' '.join (self.depotPaths),
2873                                                ' '.join (args)))
2874                 sys.exit(1)
2875
2876             self.depotPaths = sorted(args)
2877
2878         revision = ""
2879         self.users = {}
2880
2881         # Make sure no revision specifiers are used when --changesfile
2882         # is specified.
2883         bad_changesfile = False
2884         if len(self.changesFile) > 0:
2885             for p in self.depotPaths:
2886                 if p.find("@") >= 0 or p.find("#") >= 0:
2887                     bad_changesfile = True
2888                     break
2889         if bad_changesfile:
2890             die("Option --changesfile is incompatible with revision specifiers")
2891
2892         newPaths = []
2893         for p in self.depotPaths:
2894             if p.find("@") != -1:
2895                 atIdx = p.index("@")
2896                 self.changeRange = p[atIdx:]
2897                 if self.changeRange == "@all":
2898                     self.changeRange = ""
2899                 elif ',' not in self.changeRange:
2900                     revision = self.changeRange
2901                     self.changeRange = ""
2902                 p = p[:atIdx]
2903             elif p.find("#") != -1:
2904                 hashIdx = p.index("#")
2905                 revision = p[hashIdx:]
2906                 p = p[:hashIdx]
2907             elif self.previousDepotPaths == []:
2908                 # pay attention to changesfile, if given, else import
2909                 # the entire p4 tree at the head revision
2910                 if len(self.changesFile) == 0:
2911                     revision = "#head"
2912
2913             p = re.sub ("\.\.\.$", "", p)
2914             if not p.endswith("/"):
2915                 p += "/"
2916
2917             newPaths.append(p)
2918
2919         self.depotPaths = newPaths
2920
2921         # --detect-branches may change this for each branch
2922         self.branchPrefixes = self.depotPaths
2923
2924         self.loadUserMapFromCache()
2925         self.labels = {}
2926         if self.detectLabels:
2927             self.getLabels();
2928
2929         if self.detectBranches:
2930             ## FIXME - what's a P4 projectName ?
2931             self.projectName = self.guessProjectName()
2932
2933             if self.hasOrigin:
2934                 self.getBranchMappingFromGitBranches()
2935             else:
2936                 self.getBranchMapping()
2937             if self.verbose:
2938                 print "p4-git branches: %s" % self.p4BranchesInGit
2939                 print "initial parents: %s" % self.initialParents
2940             for b in self.p4BranchesInGit:
2941                 if b != "master":
2942
2943                     ## FIXME
2944                     b = b[len(self.projectName):]
2945                 self.createdBranches.add(b)
2946
2947         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2948
2949         self.importProcess = subprocess.Popen(["git", "fast-import"],
2950                                               stdin=subprocess.PIPE,
2951                                               stdout=subprocess.PIPE,
2952                                               stderr=subprocess.PIPE);
2953         self.gitOutput = self.importProcess.stdout
2954         self.gitStream = self.importProcess.stdin
2955         self.gitError = self.importProcess.stderr
2956
2957         if revision:
2958             self.importHeadRevision(revision)
2959         else:
2960             changes = []
2961
2962             if len(self.changesFile) > 0:
2963                 output = open(self.changesFile).readlines()
2964                 changeSet = set()
2965                 for line in output:
2966                     changeSet.add(int(line))
2967
2968                 for change in changeSet:
2969                     changes.append(change)
2970
2971                 changes.sort()
2972             else:
2973                 # catch "git p4 sync" with no new branches, in a repo that
2974                 # does not have any existing p4 branches
2975                 if len(args) == 0:
2976                     if not self.p4BranchesInGit:
2977                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
2978
2979                     # The default branch is master, unless --branch is used to
2980                     # specify something else.  Make sure it exists, or complain
2981                     # nicely about how to use --branch.
2982                     if not self.detectBranches:
2983                         if not branch_exists(self.branch):
2984                             if branch_arg_given:
2985                                 die("Error: branch %s does not exist." % self.branch)
2986                             else:
2987                                 die("Error: no branch %s; perhaps specify one with --branch." %
2988                                     self.branch)
2989
2990                 if self.verbose:
2991                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2992                                                               self.changeRange)
2993                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2994
2995                 if len(self.maxChanges) > 0:
2996                     changes = changes[:min(int(self.maxChanges), len(changes))]
2997
2998             if len(changes) == 0:
2999                 if not self.silent:
3000                     print "No changes to import!"
3001             else:
3002                 if not self.silent and not self.detectBranches:
3003                     print "Import destination: %s" % self.branch
3004
3005                 self.updatedBranches = set()
3006
3007                 if not self.detectBranches:
3008                     if args:
3009                         # start a new branch
3010                         self.initialParent = ""
3011                     else:
3012                         # build on a previous revision
3013                         self.initialParent = parseRevision(self.branch)
3014
3015                 self.importChanges(changes)
3016
3017                 if not self.silent:
3018                     print ""
3019                     if len(self.updatedBranches) > 0:
3020                         sys.stdout.write("Updated branches: ")
3021                         for b in self.updatedBranches:
3022                             sys.stdout.write("%s " % b)
3023                         sys.stdout.write("\n")
3024
3025         if gitConfigBool("git-p4.importLabels"):
3026             self.importLabels = True
3027
3028         if self.importLabels:
3029             p4Labels = getP4Labels(self.depotPaths)
3030             gitTags = getGitTags()
3031
3032             missingP4Labels = p4Labels - gitTags
3033             self.importP4Labels(self.gitStream, missingP4Labels)
3034
3035         self.gitStream.close()
3036         if self.importProcess.wait() != 0:
3037             die("fast-import failed: %s" % self.gitError.read())
3038         self.gitOutput.close()
3039         self.gitError.close()
3040
3041         # Cleanup temporary branches created during import
3042         if self.tempBranches != []:
3043             for branch in self.tempBranches:
3044                 read_pipe("git update-ref -d %s" % branch)
3045             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3046
3047         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3048         # a convenient shortcut refname "p4".
3049         if self.importIntoRemotes:
3050             head_ref = self.refPrefix + "HEAD"
3051             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3052                 system(["git", "symbolic-ref", head_ref, self.branch])
3053
3054         return True
3055
3056 class P4Rebase(Command):
3057     def __init__(self):
3058         Command.__init__(self)
3059         self.options = [
3060                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3061         ]
3062         self.importLabels = False
3063         self.description = ("Fetches the latest revision from perforce and "
3064                             + "rebases the current work (branch) against it")
3065
3066     def run(self, args):
3067         sync = P4Sync()
3068         sync.importLabels = self.importLabels
3069         sync.run([])
3070
3071         return self.rebase()
3072
3073     def rebase(self):
3074         if os.system("git update-index --refresh") != 0:
3075             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3076         if len(read_pipe("git diff-index HEAD --")) > 0:
3077             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3078
3079         [upstream, settings] = findUpstreamBranchPoint()
3080         if len(upstream) == 0:
3081             die("Cannot find upstream branchpoint for rebase")
3082
3083         # the branchpoint may be p4/foo~3, so strip off the parent
3084         upstream = re.sub("~[0-9]+$", "", upstream)
3085
3086         print "Rebasing the current branch onto %s" % upstream
3087         oldHead = read_pipe("git rev-parse HEAD").strip()
3088         system("git rebase %s" % upstream)
3089         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3090         return True
3091
3092 class P4Clone(P4Sync):
3093     def __init__(self):
3094         P4Sync.__init__(self)
3095         self.description = "Creates a new git repository and imports from Perforce into it"
3096         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3097         self.options += [
3098             optparse.make_option("--destination", dest="cloneDestination",
3099                                  action='store', default=None,
3100                                  help="where to leave result of the clone"),
3101             optparse.make_option("-/", dest="cloneExclude",
3102                                  action="append", type="string",
3103                                  help="exclude depot path"),
3104             optparse.make_option("--bare", dest="cloneBare",
3105                                  action="store_true", default=False),
3106         ]
3107         self.cloneDestination = None
3108         self.needsGit = False
3109         self.cloneBare = False
3110
3111     # This is required for the "append" cloneExclude action
3112     def ensure_value(self, attr, value):
3113         if not hasattr(self, attr) or getattr(self, attr) is None:
3114             setattr(self, attr, value)
3115         return getattr(self, attr)
3116
3117     def defaultDestination(self, args):
3118         ## TODO: use common prefix of args?
3119         depotPath = args[0]
3120         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3121         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3122         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3123         depotDir = re.sub(r"/$", "", depotDir)
3124         return os.path.split(depotDir)[1]
3125
3126     def run(self, args):
3127         if len(args) < 1:
3128             return False
3129
3130         if self.keepRepoPath and not self.cloneDestination:
3131             sys.stderr.write("Must specify destination for --keep-path\n")
3132             sys.exit(1)
3133
3134         depotPaths = args
3135
3136         if not self.cloneDestination and len(depotPaths) > 1:
3137             self.cloneDestination = depotPaths[-1]
3138             depotPaths = depotPaths[:-1]
3139
3140         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3141         for p in depotPaths:
3142             if not p.startswith("//"):
3143                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3144                 return False
3145
3146         if not self.cloneDestination:
3147             self.cloneDestination = self.defaultDestination(args)
3148
3149         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3150
3151         if not os.path.exists(self.cloneDestination):
3152             os.makedirs(self.cloneDestination)
3153         chdir(self.cloneDestination)
3154
3155         init_cmd = [ "git", "init" ]
3156         if self.cloneBare:
3157             init_cmd.append("--bare")
3158         retcode = subprocess.call(init_cmd)
3159         if retcode:
3160             raise CalledProcessError(retcode, init_cmd)
3161
3162         if not P4Sync.run(self, depotPaths):
3163             return False
3164
3165         # create a master branch and check out a work tree
3166         if gitBranchExists(self.branch):
3167             system([ "git", "branch", "master", self.branch ])
3168             if not self.cloneBare:
3169                 system([ "git", "checkout", "-f" ])
3170         else:
3171             print 'Not checking out any branch, use ' \
3172                   '"git checkout -q -b master <branch>"'
3173
3174         # auto-set this variable if invoked with --use-client-spec
3175         if self.useClientSpec_from_options:
3176             system("git config --bool git-p4.useclientspec true")
3177
3178         return True
3179
3180 class P4Branches(Command):
3181     def __init__(self):
3182         Command.__init__(self)
3183         self.options = [ ]
3184         self.description = ("Shows the git branches that hold imports and their "
3185                             + "corresponding perforce depot paths")
3186         self.verbose = False
3187
3188     def run(self, args):
3189         if originP4BranchesExist():
3190             createOrUpdateBranchesFromOrigin()
3191
3192         cmdline = "git rev-parse --symbolic "
3193         cmdline += " --remotes"
3194
3195         for line in read_pipe_lines(cmdline):
3196             line = line.strip()
3197
3198             if not line.startswith('p4/') or line == "p4/HEAD":
3199                 continue
3200             branch = line
3201
3202             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3203             settings = extractSettingsGitLog(log)
3204
3205             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3206         return True
3207
3208 class HelpFormatter(optparse.IndentedHelpFormatter):
3209     def __init__(self):
3210         optparse.IndentedHelpFormatter.__init__(self)
3211
3212     def format_description(self, description):
3213         if description:
3214             return description + "\n"
3215         else:
3216             return ""
3217
3218 def printUsage(commands):
3219     print "usage: %s <command> [options]" % sys.argv[0]
3220     print ""
3221     print "valid commands: %s" % ", ".join(commands)
3222     print ""
3223     print "Try %s <command> --help for command specific help." % sys.argv[0]
3224     print ""
3225
3226 commands = {
3227     "debug" : P4Debug,
3228     "submit" : P4Submit,
3229     "commit" : P4Submit,
3230     "sync" : P4Sync,
3231     "rebase" : P4Rebase,
3232     "clone" : P4Clone,
3233     "rollback" : P4RollBack,
3234     "branches" : P4Branches
3235 }
3236
3237
3238 def main():
3239     if len(sys.argv[1:]) == 0:
3240         printUsage(commands.keys())
3241         sys.exit(2)
3242
3243     cmdName = sys.argv[1]
3244     try:
3245         klass = commands[cmdName]
3246         cmd = klass()
3247     except KeyError:
3248         print "unknown command %s" % cmdName
3249         print ""
3250         printUsage(commands.keys())
3251         sys.exit(2)
3252
3253     options = cmd.options
3254     cmd.gitdir = os.environ.get("GIT_DIR", None)
3255
3256     args = sys.argv[2:]
3257
3258     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3259     if cmd.needsGit:
3260         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3261
3262     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3263                                    options,
3264                                    description = cmd.description,
3265                                    formatter = HelpFormatter())
3266
3267     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3268     global verbose
3269     verbose = cmd.verbose
3270     if cmd.needsGit:
3271         if cmd.gitdir == None:
3272             cmd.gitdir = os.path.abspath(".git")
3273             if not isValidGitDir(cmd.gitdir):
3274                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3275                 if os.path.exists(cmd.gitdir):
3276                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3277                     if len(cdup) > 0:
3278                         chdir(cdup);
3279
3280         if not isValidGitDir(cmd.gitdir):
3281             if isValidGitDir(cmd.gitdir + "/.git"):
3282                 cmd.gitdir += "/.git"
3283             else:
3284                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3285
3286         os.environ["GIT_DIR"] = cmd.gitdir
3287
3288     if not cmd.run(args):
3289         parser.print_help()
3290         sys.exit(2)
3291
3292
3293 if __name__ == '__main__':
3294     main()