git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10 import sys
  11 if sys.hexversion < 0x02040000:
  12     # The limiter is the subprocess module
  13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14     sys.exit(1)
  15 import os
  16 import optparse
  17 import marshal
  18 import subprocess
  19 import tempfile
  20 import time
  21 import platform
  22 import re
  23 import shutil
  24 import stat
  25 import zipfile
  26 import zlib
  27 import ctypes
  28 import errno
  29
  30 try:
  31     from subprocess import CalledProcessError
  32 except ImportError:
  33     # from python2.7:subprocess.py
  34     # Exception classes used by this module.
  35     class CalledProcessError(Exception):
  36         """This exception is raised when a process run by check_call() returns
  37         a non-zero exit status.  The exit status will be stored in the
  38         returncode attribute."""
  39         def __init__(self, returncode, cmd):
  40             self.returncode = returncode
  41             self.cmd = cmd
  42         def __str__(self):
  43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  44
  45 verbose = False
  46
  47 # Only labels/tags matching this will be imported/exported
  48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  49
  50 # Grab changes in blocks of this many revisions, unless otherwise requested
  51 defaultBlockSize = 512
  52
  53 def p4_build_cmd(cmd):
  54     """Build a suitable p4 command line.
  55
  56     This consolidates building and returning a p4 command line into one
  57     location. It means that hooking into the environment, or other configuration
  58     can be done more easily.
  59     """
  60     real_cmd = ["p4"]
  61
  62     user = gitConfig("git-p4.user")
  63     if len(user) > 0:
  64         real_cmd += ["-u",user]
  65
  66     password = gitConfig("git-p4.password")
  67     if len(password) > 0:
  68         real_cmd += ["-P", password]
  69
  70     port = gitConfig("git-p4.port")
  71     if len(port) > 0:
  72         real_cmd += ["-p", port]
  73
  74     host = gitConfig("git-p4.host")
  75     if len(host) > 0:
  76         real_cmd += ["-H", host]
  77
  78     client = gitConfig("git-p4.client")
  79     if len(client) > 0:
  80         real_cmd += ["-c", client]
  81
  82     retries = gitConfigInt("git-p4.retries")
  83     if retries is None:
  84         # Perform 3 retries by default
  85         retries = 3
  86     if retries > 0:
  87         # Provide a way to not pass this option by setting git-p4.retries to 0
  88         real_cmd += ["-r", str(retries)]
  89
  90     if isinstance(cmd,basestring):
  91         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  92     else:
  93         real_cmd += cmd
  94     return real_cmd
  95
  96 def git_dir(path):
  97     """ Return TRUE if the given path is a git directory (/path/to/dir/.git).
  98         This won't automatically add ".git" to a directory.
  99     """
 100     d = read_pipe(["git", "--git-dir", path, "rev-parse", "--git-dir"], True).strip()
 101     if not d or len(d) == 0:
 102         return None
 103     else:
 104         return d
 105
 106 def chdir(path, is_client_path=False):
 107     """Do chdir to the given path, and set the PWD environment
 108        variable for use by P4.  It does not look at getcwd() output.
 109        Since we're not using the shell, it is necessary to set the
 110        PWD environment variable explicitly.
 111
 112        Normally, expand the path to force it to be absolute.  This
 113        addresses the use of relative path names inside P4 settings,
 114        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
 115        as given; it looks for .p4config using PWD.
 116
 117        If is_client_path, the path was handed to us directly by p4,
 118        and may be a symbolic link.  Do not call os.getcwd() in this
 119        case, because it will cause p4 to think that PWD is not inside
 120        the client path.
 121        """
 122
 123     os.chdir(path)
 124     if not is_client_path:
 125         path = os.getcwd()
 126     os.environ['PWD'] = path
 127
 128 def calcDiskFree():
 129     """Return free space in bytes on the disk of the given dirname."""
 130     if platform.system() == 'Windows':
 131         free_bytes = ctypes.c_ulonglong(0)
 132         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
 133         return free_bytes.value
 134     else:
 135         st = os.statvfs(os.getcwd())
 136         return st.f_bavail * st.f_frsize
 137
 138 def die(msg):
 139     if verbose:
 140         raise Exception(msg)
 141     else:
 142         sys.stderr.write(msg + "\n")
 143         sys.exit(1)
 144
 145 def write_pipe(c, stdin):
 146     if verbose:
 147         sys.stderr.write('Writing pipe: %s\n' % str(c))
 148
 149     expand = isinstance(c,basestring)
 150     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 151     pipe = p.stdin
 152     val = pipe.write(stdin)
 153     pipe.close()
 154     if p.wait():
 155         die('Command failed: %s' % str(c))
 156
 157     return val
 158
 159 def p4_write_pipe(c, stdin):
 160     real_cmd = p4_build_cmd(c)
 161     return write_pipe(real_cmd, stdin)
 162
 163 def read_pipe_full(c):
 164     """ Read output from  command. Returns a tuple
 165         of the return status, stdout text and stderr
 166         text.
 167     """
 168     if verbose:
 169         sys.stderr.write('Reading pipe: %s\n' % str(c))
 170
 171     expand = isinstance(c,basestring)
 172     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
 173     (out, err) = p.communicate()
 174     return (p.returncode, out, err)
 175
 176 def read_pipe(c, ignore_error=False):
 177     """ Read output from  command. Returns the output text on
 178         success. On failure, terminates execution, unless
 179         ignore_error is True, when it returns an empty string.
 180     """
 181     (retcode, out, err) = read_pipe_full(c)
 182     if retcode != 0:
 183         if ignore_error:
 184             out = ""
 185         else:
 186             die('Command failed: %s\nError: %s' % (str(c), err))
 187     return out
 188
 189 def read_pipe_text(c):
 190     """ Read output from a command with trailing whitespace stripped.
 191         On error, returns None.
 192     """
 193     (retcode, out, err) = read_pipe_full(c)
 194     if retcode != 0:
 195         return None
 196     else:
 197         return out.rstrip()
 198
 199 def p4_read_pipe(c, ignore_error=False):
 200     real_cmd = p4_build_cmd(c)
 201     return read_pipe(real_cmd, ignore_error)
 202
 203 def read_pipe_lines(c):
 204     if verbose:
 205         sys.stderr.write('Reading pipe: %s\n' % str(c))
 206
 207     expand = isinstance(c, basestring)
 208     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 209     pipe = p.stdout
 210     val = pipe.readlines()
 211     if pipe.close() or p.wait():
 212         die('Command failed: %s' % str(c))
 213
 214     return val
 215
 216 def p4_read_pipe_lines(c):
 217     """Specifically invoke p4 on the command supplied. """
 218     real_cmd = p4_build_cmd(c)
 219     return read_pipe_lines(real_cmd)
 220
 221 def p4_has_command(cmd):
 222     """Ask p4 for help on this command.  If it returns an error, the
 223        command does not exist in this version of p4."""
 224     real_cmd = p4_build_cmd(["help", cmd])
 225     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 226                                    stderr=subprocess.PIPE)
 227     p.communicate()
 228     return p.returncode == 0
 229
 230 def p4_has_move_command():
 231     """See if the move command exists, that it supports -k, and that
 232        it has not been administratively disabled.  The arguments
 233        must be correct, but the filenames do not have to exist.  Use
 234        ones with wildcards so even if they exist, it will fail."""
 235
 236     if not p4_has_command("move"):
 237         return False
 238     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 239     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 240     (out, err) = p.communicate()
 241     # return code will be 1 in either case
 242     if err.find("Invalid option") >= 0:
 243         return False
 244     if err.find("disabled") >= 0:
 245         return False
 246     # assume it failed because @... was invalid changelist
 247     return True
 248
 249 def system(cmd, ignore_error=False):
 250     expand = isinstance(cmd,basestring)
 251     if verbose:
 252         sys.stderr.write("executing %s\n" % str(cmd))
 253     retcode = subprocess.call(cmd, shell=expand)
 254     if retcode and not ignore_error:
 255         raise CalledProcessError(retcode, cmd)
 256
 257     return retcode
 258
 259 def p4_system(cmd):
 260     """Specifically invoke p4 as the system command. """
 261     real_cmd = p4_build_cmd(cmd)
 262     expand = isinstance(real_cmd, basestring)
 263     retcode = subprocess.call(real_cmd, shell=expand)
 264     if retcode:
 265         raise CalledProcessError(retcode, real_cmd)
 266
 267 _p4_version_string = None
 268 def p4_version_string():
 269     """Read the version string, showing just the last line, which
 270        hopefully is the interesting version bit.
 271
 272        $ p4 -V
 273        Perforce - The Fast Software Configuration Management System.
 274        Copyright 1995-2011 Perforce Software.  All rights reserved.
 275        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 276     """
 277     global _p4_version_string
 278     if not _p4_version_string:
 279         a = p4_read_pipe_lines(["-V"])
 280         _p4_version_string = a[-1].rstrip()
 281     return _p4_version_string
 282
 283 def p4_integrate(src, dest):
 284     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 285
 286 def p4_sync(f, *options):
 287     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 288
 289 def p4_add(f):
 290     # forcibly add file names with wildcards
 291     if wildcard_present(f):
 292         p4_system(["add", "-f", f])
 293     else:
 294         p4_system(["add", f])
 295
 296 def p4_delete(f):
 297     p4_system(["delete", wildcard_encode(f)])
 298
 299 def p4_edit(f, *options):
 300     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
 301
 302 def p4_revert(f):
 303     p4_system(["revert", wildcard_encode(f)])
 304
 305 def p4_reopen(type, f):
 306     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 307
 308 def p4_reopen_in_change(changelist, files):
 309     cmd = ["reopen", "-c", str(changelist)] + files
 310     p4_system(cmd)
 311
 312 def p4_move(src, dest):
 313     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 314
 315 def p4_last_change():
 316     results = p4CmdList(["changes", "-m", "1"])
 317     return int(results[0]['change'])
 318
 319 def p4_describe(change):
 320     """Make sure it returns a valid result by checking for
 321        the presence of field "time".  Return a dict of the
 322        results."""
 323
 324     ds = p4CmdList(["describe", "-s", str(change)])
 325     if len(ds) != 1:
 326         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 327
 328     d = ds[0]
 329
 330     if "p4ExitCode" in d:
 331         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 332                                                       str(d)))
 333     if "code" in d:
 334         if d["code"] == "error":
 335             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 336
 337     if "time" not in d:
 338         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 339
 340     return d
 341
 342 #
 343 # Canonicalize the p4 type and return a tuple of the
 344 # base type, plus any modifiers.  See "p4 help filetypes"
 345 # for a list and explanation.
 346 #
 347 def split_p4_type(p4type):
 348
 349     p4_filetypes_historical = {
 350         "ctempobj": "binary+Sw",
 351         "ctext": "text+C",
 352         "cxtext": "text+Cx",
 353         "ktext": "text+k",
 354         "kxtext": "text+kx",
 355         "ltext": "text+F",
 356         "tempobj": "binary+FSw",
 357         "ubinary": "binary+F",
 358         "uresource": "resource+F",
 359         "uxbinary": "binary+Fx",
 360         "xbinary": "binary+x",
 361         "xltext": "text+Fx",
 362         "xtempobj": "binary+Swx",
 363         "xtext": "text+x",
 364         "xunicode": "unicode+x",
 365         "xutf16": "utf16+x",
 366     }
 367     if p4type in p4_filetypes_historical:
 368         p4type = p4_filetypes_historical[p4type]
 369     mods = ""
 370     s = p4type.split("+")
 371     base = s[0]
 372     mods = ""
 373     if len(s) > 1:
 374         mods = s[1]
 375     return (base, mods)
 376
 377 #
 378 # return the raw p4 type of a file (text, text+ko, etc)
 379 #
 380 def p4_type(f):
 381     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 382     return results[0]['headType']
 383
 384 #
 385 # Given a type base and modifier, return a regexp matching
 386 # the keywords that can be expanded in the file
 387 #
 388 def p4_keywords_regexp_for_type(base, type_mods):
 389     if base in ("text", "unicode", "binary"):
 390         kwords = None
 391         if "ko" in type_mods:
 392             kwords = 'Id|Header'
 393         elif "k" in type_mods:
 394             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 395         else:
 396             return None
 397         pattern = r"""
 398             \$              # Starts with a dollar, followed by...
 399             (%s)            # one of the keywords, followed by...
 400             (:[^$\n]+)?     # possibly an old expansion, followed by...
 401             \$              # another dollar
 402             """ % kwords
 403         return pattern
 404     else:
 405         return None
 406
 407 #
 408 # Given a file, return a regexp matching the possible
 409 # RCS keywords that will be expanded, or None for files
 410 # with kw expansion turned off.
 411 #
 412 def p4_keywords_regexp_for_file(file):
 413     if not os.path.exists(file):
 414         return None
 415     else:
 416         (type_base, type_mods) = split_p4_type(p4_type(file))
 417         return p4_keywords_regexp_for_type(type_base, type_mods)
 418
 419 def setP4ExecBit(file, mode):
 420     # Reopens an already open file and changes the execute bit to match
 421     # the execute bit setting in the passed in mode.
 422
 423     p4Type = "+x"
 424
 425     if not isModeExec(mode):
 426         p4Type = getP4OpenedType(file)
 427         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 428         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 429         if p4Type[-1] == "+":
 430             p4Type = p4Type[0:-1]
 431
 432     p4_reopen(p4Type, file)
 433
 434 def getP4OpenedType(file):
 435     # Returns the perforce file type for the given file.
 436
 437     result = p4_read_pipe(["opened", wildcard_encode(file)])
 438     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 439     if match:
 440         return match.group(1)
 441     else:
 442         die("Could not determine file type for %s (result: '%s')" % (file, result))
 443
 444 # Return the set of all p4 labels
 445 def getP4Labels(depotPaths):
 446     labels = set()
 447     if isinstance(depotPaths,basestring):
 448         depotPaths = [depotPaths]
 449
 450     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 451         label = l['label']
 452         labels.add(label)
 453
 454     return labels
 455
 456 # Return the set of all git tags
 457 def getGitTags():
 458     gitTags = set()
 459     for line in read_pipe_lines(["git", "tag"]):
 460         tag = line.strip()
 461         gitTags.add(tag)
 462     return gitTags
 463
 464 def diffTreePattern():
 465     # This is a simple generator for the diff tree regex pattern. This could be
 466     # a class variable if this and parseDiffTreeEntry were a part of a class.
 467     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 468     while True:
 469         yield pattern
 470
 471 def parseDiffTreeEntry(entry):
 472     """Parses a single diff tree entry into its component elements.
 473
 474     See git-diff-tree(1) manpage for details about the format of the diff
 475     output. This method returns a dictionary with the following elements:
 476
 477     src_mode - The mode of the source file
 478     dst_mode - The mode of the destination file
 479     src_sha1 - The sha1 for the source file
 480     dst_sha1 - The sha1 fr the destination file
 481     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 482     status_score - The score for the status (applicable for 'C' and 'R'
 483                    statuses). This is None if there is no score.
 484     src - The path for the source file.
 485     dst - The path for the destination file. This is only present for
 486           copy or renames. If it is not present, this is None.
 487
 488     If the pattern is not matched, None is returned."""
 489
 490     match = diffTreePattern().next().match(entry)
 491     if match:
 492         return {
 493             'src_mode': match.group(1),
 494             'dst_mode': match.group(2),
 495             'src_sha1': match.group(3),
 496             'dst_sha1': match.group(4),
 497             'status': match.group(5),
 498             'status_score': match.group(6),
 499             'src': match.group(7),
 500             'dst': match.group(10)
 501         }
 502     return None
 503
 504 def isModeExec(mode):
 505     # Returns True if the given git mode represents an executable file,
 506     # otherwise False.
 507     return mode[-3:] == "755"
 508
 509 def isModeExecChanged(src_mode, dst_mode):
 510     return isModeExec(src_mode) != isModeExec(dst_mode)
 511
 512 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 513
 514     if isinstance(cmd,basestring):
 515         cmd = "-G " + cmd
 516         expand = True
 517     else:
 518         cmd = ["-G"] + cmd
 519         expand = False
 520
 521     cmd = p4_build_cmd(cmd)
 522     if verbose:
 523         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 524
 525     # Use a temporary file to avoid deadlocks without
 526     # subprocess.communicate(), which would put another copy
 527     # of stdout into memory.
 528     stdin_file = None
 529     if stdin is not None:
 530         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 531         if isinstance(stdin,basestring):
 532             stdin_file.write(stdin)
 533         else:
 534             for i in stdin:
 535                 stdin_file.write(i + '\n')
 536         stdin_file.flush()
 537         stdin_file.seek(0)
 538
 539     p4 = subprocess.Popen(cmd,
 540                           shell=expand,
 541                           stdin=stdin_file,
 542                           stdout=subprocess.PIPE)
 543
 544     result = []
 545     try:
 546         while True:
 547             entry = marshal.load(p4.stdout)
 548             if cb is not None:
 549                 cb(entry)
 550             else:
 551                 result.append(entry)
 552     except EOFError:
 553         pass
 554     exitCode = p4.wait()
 555     if exitCode != 0:
 556         entry = {}
 557         entry["p4ExitCode"] = exitCode
 558         result.append(entry)
 559
 560     return result
 561
 562 def p4Cmd(cmd):
 563     list = p4CmdList(cmd)
 564     result = {}
 565     for entry in list:
 566         result.update(entry)
 567     return result;
 568
 569 def p4Where(depotPath):
 570     if not depotPath.endswith("/"):
 571         depotPath += "/"
 572     depotPathLong = depotPath + "..."
 573     outputList = p4CmdList(["where", depotPathLong])
 574     output = None
 575     for entry in outputList:
 576         if "depotFile" in entry:
 577             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 578             # The base path always ends with "/...".
 579             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 580                 output = entry
 581                 break
 582         elif "data" in entry:
 583             data = entry.get("data")
 584             space = data.find(" ")
 585             if data[:space] == depotPath:
 586                 output = entry
 587                 break
 588     if output == None:
 589         return ""
 590     if output["code"] == "error":
 591         return ""
 592     clientPath = ""
 593     if "path" in output:
 594         clientPath = output.get("path")
 595     elif "data" in output:
 596         data = output.get("data")
 597         lastSpace = data.rfind(" ")
 598         clientPath = data[lastSpace + 1:]
 599
 600     if clientPath.endswith("..."):
 601         clientPath = clientPath[:-3]
 602     return clientPath
 603
 604 def currentGitBranch():
 605     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
 606     if retcode != 0:
 607         # on a detached head
 608         return None
 609     else:
 610         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
 611
 612 def isValidGitDir(path):
 613     return git_dir(path) != None
 614
 615 def parseRevision(ref):
 616     return read_pipe("git rev-parse %s" % ref).strip()
 617
 618 def branchExists(ref):
 619     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 620                      ignore_error=True)
 621     return len(rev) > 0
 622
 623 def extractLogMessageFromGitCommit(commit):
 624     logMessage = ""
 625
 626     ## fixme: title is first line of commit, not 1st paragraph.
 627     foundTitle = False
 628     for log in read_pipe_lines("git cat-file commit %s" % commit):
 629        if not foundTitle:
 630            if len(log) == 1:
 631                foundTitle = True
 632            continue
 633
 634        logMessage += log
 635     return logMessage
 636
 637 def extractSettingsGitLog(log):
 638     values = {}
 639     for line in log.split("\n"):
 640         line = line.strip()
 641         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 642         if not m:
 643             continue
 644
 645         assignments = m.group(1).split (':')
 646         for a in assignments:
 647             vals = a.split ('=')
 648             key = vals[0].strip()
 649             val = ('='.join (vals[1:])).strip()
 650             if val.endswith ('\"') and val.startswith('"'):
 651                 val = val[1:-1]
 652
 653             values[key] = val
 654
 655     paths = values.get("depot-paths")
 656     if not paths:
 657         paths = values.get("depot-path")
 658     if paths:
 659         values['depot-paths'] = paths.split(',')
 660     return values
 661
 662 def gitBranchExists(branch):
 663     proc = subprocess.Popen(["git", "rev-parse", branch],
 664                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 665     return proc.wait() == 0;
 666
 667 _gitConfig = {}
 668
 669 def gitConfig(key, typeSpecifier=None):
 670     if not _gitConfig.has_key(key):
 671         cmd = [ "git", "config" ]
 672         if typeSpecifier:
 673             cmd += [ typeSpecifier ]
 674         cmd += [ key ]
 675         s = read_pipe(cmd, ignore_error=True)
 676         _gitConfig[key] = s.strip()
 677     return _gitConfig[key]
 678
 679 def gitConfigBool(key):
 680     """Return a bool, using git config --bool.  It is True only if the
 681        variable is set to true, and False if set to false or not present
 682        in the config."""
 683
 684     if not _gitConfig.has_key(key):
 685         _gitConfig[key] = gitConfig(key, '--bool') == "true"
 686     return _gitConfig[key]
 687
 688 def gitConfigInt(key):
 689     if not _gitConfig.has_key(key):
 690         cmd = [ "git", "config", "--int", key ]
 691         s = read_pipe(cmd, ignore_error=True)
 692         v = s.strip()
 693         try:
 694             _gitConfig[key] = int(gitConfig(key, '--int'))
 695         except ValueError:
 696             _gitConfig[key] = None
 697     return _gitConfig[key]
 698
 699 def gitConfigList(key):
 700     if not _gitConfig.has_key(key):
 701         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 702         _gitConfig[key] = s.strip().splitlines()
 703         if _gitConfig[key] == ['']:
 704             _gitConfig[key] = []
 705     return _gitConfig[key]
 706
 707 def p4BranchesInGit(branchesAreInRemotes=True):
 708     """Find all the branches whose names start with "p4/", looking
 709        in remotes or heads as specified by the argument.  Return
 710        a dictionary of { branch: revision } for each one found.
 711        The branch names are the short names, without any
 712        "p4/" prefix."""
 713
 714     branches = {}
 715
 716     cmdline = "git rev-parse --symbolic "
 717     if branchesAreInRemotes:
 718         cmdline += "--remotes"
 719     else:
 720         cmdline += "--branches"
 721
 722     for line in read_pipe_lines(cmdline):
 723         line = line.strip()
 724
 725         # only import to p4/
 726         if not line.startswith('p4/'):
 727             continue
 728         # special symbolic ref to p4/master
 729         if line == "p4/HEAD":
 730             continue
 731
 732         # strip off p4/ prefix
 733         branch = line[len("p4/"):]
 734
 735         branches[branch] = parseRevision(line)
 736
 737     return branches
 738
 739 def branch_exists(branch):
 740     """Make sure that the given ref name really exists."""
 741
 742     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 743     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 744     out, _ = p.communicate()
 745     if p.returncode:
 746         return False
 747     # expect exactly one line of output: the branch name
 748     return out.rstrip() == branch
 749
 750 def findUpstreamBranchPoint(head = "HEAD"):
 751     branches = p4BranchesInGit()
 752     # map from depot-path to branch name
 753     branchByDepotPath = {}
 754     for branch in branches.keys():
 755         tip = branches[branch]
 756         log = extractLogMessageFromGitCommit(tip)
 757         settings = extractSettingsGitLog(log)
 758         if settings.has_key("depot-paths"):
 759             paths = ",".join(settings["depot-paths"])
 760             branchByDepotPath[paths] = "remotes/p4/" + branch
 761
 762     settings = None
 763     parent = 0
 764     while parent < 65535:
 765         commit = head + "~%s" % parent
 766         log = extractLogMessageFromGitCommit(commit)
 767         settings = extractSettingsGitLog(log)
 768         if settings.has_key("depot-paths"):
 769             paths = ",".join(settings["depot-paths"])
 770             if branchByDepotPath.has_key(paths):
 771                 return [branchByDepotPath[paths], settings]
 772
 773         parent = parent + 1
 774
 775     return ["", settings]
 776
 777 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 778     if not silent:
 779         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 780                % localRefPrefix)
 781
 782     originPrefix = "origin/p4/"
 783
 784     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 785         line = line.strip()
 786         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 787             continue
 788
 789         headName = line[len(originPrefix):]
 790         remoteHead = localRefPrefix + headName
 791         originHead = line
 792
 793         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 794         if (not original.has_key('depot-paths')
 795             or not original.has_key('change')):
 796             continue
 797
 798         update = False
 799         if not gitBranchExists(remoteHead):
 800             if verbose:
 801                 print "creating %s" % remoteHead
 802             update = True
 803         else:
 804             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 805             if settings.has_key('change') > 0:
 806                 if settings['depot-paths'] == original['depot-paths']:
 807                     originP4Change = int(original['change'])
 808                     p4Change = int(settings['change'])
 809                     if originP4Change > p4Change:
 810                         print ("%s (%s) is newer than %s (%s). "
 811                                "Updating p4 branch from origin."
 812                                % (originHead, originP4Change,
 813                                   remoteHead, p4Change))
 814                         update = True
 815                 else:
 816                     print ("Ignoring: %s was imported from %s while "
 817                            "%s was imported from %s"
 818                            % (originHead, ','.join(original['depot-paths']),
 819                               remoteHead, ','.join(settings['depot-paths'])))
 820
 821         if update:
 822             system("git update-ref %s %s" % (remoteHead, originHead))
 823
 824 def originP4BranchesExist():
 825         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 826
 827
 828 def p4ParseNumericChangeRange(parts):
 829     changeStart = int(parts[0][1:])
 830     if parts[1] == '#head':
 831         changeEnd = p4_last_change()
 832     else:
 833         changeEnd = int(parts[1])
 834
 835     return (changeStart, changeEnd)
 836
 837 def chooseBlockSize(blockSize):
 838     if blockSize:
 839         return blockSize
 840     else:
 841         return defaultBlockSize
 842
 843 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 844     assert depotPaths
 845
 846     # Parse the change range into start and end. Try to find integer
 847     # revision ranges as these can be broken up into blocks to avoid
 848     # hitting server-side limits (maxrows, maxscanresults). But if
 849     # that doesn't work, fall back to using the raw revision specifier
 850     # strings, without using block mode.
 851
 852     if changeRange is None or changeRange == '':
 853         changeStart = 1
 854         changeEnd = p4_last_change()
 855         block_size = chooseBlockSize(requestedBlockSize)
 856     else:
 857         parts = changeRange.split(',')
 858         assert len(parts) == 2
 859         try:
 860             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 861             block_size = chooseBlockSize(requestedBlockSize)
 862         except:
 863             changeStart = parts[0][1:]
 864             changeEnd = parts[1]
 865             if requestedBlockSize:
 866                 die("cannot use --changes-block-size with non-numeric revisions")
 867             block_size = None
 868
 869     changes = set()
 870
 871     # Retrieve changes a block at a time, to prevent running
 872     # into a MaxResults/MaxScanRows error from the server.
 873
 874     while True:
 875         cmd = ['changes']
 876
 877         if block_size:
 878             end = min(changeEnd, changeStart + block_size)
 879             revisionRange = "%d,%d" % (changeStart, end)
 880         else:
 881             revisionRange = "%s,%s" % (changeStart, changeEnd)
 882
 883         for p in depotPaths:
 884             cmd += ["%s...@%s" % (p, revisionRange)]
 885
 886         # Insert changes in chronological order
 887         for line in reversed(p4_read_pipe_lines(cmd)):
 888             changes.add(int(line.split(" ")[1]))
 889
 890         if not block_size:
 891             break
 892
 893         if end >= changeEnd:
 894             break
 895
 896         changeStart = end + 1
 897
 898     changes = sorted(changes)
 899     return changes
 900
 901 def p4PathStartsWith(path, prefix):
 902     # This method tries to remedy a potential mixed-case issue:
 903     #
 904     # If UserA adds  //depot/DirA/file1
 905     # and UserB adds //depot/dira/file2
 906     #
 907     # we may or may not have a problem. If you have core.ignorecase=true,
 908     # we treat DirA and dira as the same directory
 909     if gitConfigBool("core.ignorecase"):
 910         return path.lower().startswith(prefix.lower())
 911     return path.startswith(prefix)
 912
 913 def getClientSpec():
 914     """Look at the p4 client spec, create a View() object that contains
 915        all the mappings, and return it."""
 916
 917     specList = p4CmdList("client -o")
 918     if len(specList) != 1:
 919         die('Output from "client -o" is %d lines, expecting 1' %
 920             len(specList))
 921
 922     # dictionary of all client parameters
 923     entry = specList[0]
 924
 925     # the //client/ name
 926     client_name = entry["Client"]
 927
 928     # just the keys that start with "View"
 929     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 930
 931     # hold this new View
 932     view = View(client_name)
 933
 934     # append the lines, in order, to the view
 935     for view_num in range(len(view_keys)):
 936         k = "View%d" % view_num
 937         if k not in view_keys:
 938             die("Expected view key %s missing" % k)
 939         view.append(entry[k])
 940
 941     return view
 942
 943 def getClientRoot():
 944     """Grab the client directory."""
 945
 946     output = p4CmdList("client -o")
 947     if len(output) != 1:
 948         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 949
 950     entry = output[0]
 951     if "Root" not in entry:
 952         die('Client has no "Root"')
 953
 954     return entry["Root"]
 955
 956 #
 957 # P4 wildcards are not allowed in filenames.  P4 complains
 958 # if you simply add them, but you can force it with "-f", in
 959 # which case it translates them into %xx encoding internally.
 960 #
 961 def wildcard_decode(path):
 962     # Search for and fix just these four characters.  Do % last so
 963     # that fixing it does not inadvertently create new %-escapes.
 964     # Cannot have * in a filename in windows; untested as to
 965     # what p4 would do in such a case.
 966     if not platform.system() == "Windows":
 967         path = path.replace("%2A", "*")
 968     path = path.replace("%23", "#") \
 969                .replace("%40", "@") \
 970                .replace("%25", "%")
 971     return path
 972
 973 def wildcard_encode(path):
 974     # do % first to avoid double-encoding the %s introduced here
 975     path = path.replace("%", "%25") \
 976                .replace("*", "%2A") \
 977                .replace("#", "%23") \
 978                .replace("@", "%40")
 979     return path
 980
 981 def wildcard_present(path):
 982     m = re.search("[*#@%]", path)
 983     return m is not None
 984
 985 class LargeFileSystem(object):
 986     """Base class for large file system support."""
 987
 988     def __init__(self, writeToGitStream):
 989         self.largeFiles = set()
 990         self.writeToGitStream = writeToGitStream
 991
 992     def generatePointer(self, cloneDestination, contentFile):
 993         """Return the content of a pointer file that is stored in Git instead of
 994            the actual content."""
 995         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
 996
 997     def pushFile(self, localLargeFile):
 998         """Push the actual content which is not stored in the Git repository to
 999            a server."""
1000         assert False, "Method 'pushFile' required in " + self.__class__.__name__
1001
1002     def hasLargeFileExtension(self, relPath):
1003         return reduce(
1004             lambda a, b: a or b,
1005             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
1006             False
1007         )
1008
1009     def generateTempFile(self, contents):
1010         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1011         for d in contents:
1012             contentFile.write(d)
1013         contentFile.close()
1014         return contentFile.name
1015
1016     def exceedsLargeFileThreshold(self, relPath, contents):
1017         if gitConfigInt('git-p4.largeFileThreshold'):
1018             contentsSize = sum(len(d) for d in contents)
1019             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
1020                 return True
1021         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
1022             contentsSize = sum(len(d) for d in contents)
1023             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
1024                 return False
1025             contentTempFile = self.generateTempFile(contents)
1026             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
1027             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
1028             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
1029             zf.close()
1030             compressedContentsSize = zf.infolist()[0].compress_size
1031             os.remove(contentTempFile)
1032             os.remove(compressedContentFile.name)
1033             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
1034                 return True
1035         return False
1036
1037     def addLargeFile(self, relPath):
1038         self.largeFiles.add(relPath)
1039
1040     def removeLargeFile(self, relPath):
1041         self.largeFiles.remove(relPath)
1042
1043     def isLargeFile(self, relPath):
1044         return relPath in self.largeFiles
1045
1046     def processContent(self, git_mode, relPath, contents):
1047         """Processes the content of git fast import. This method decides if a
1048            file is stored in the large file system and handles all necessary
1049            steps."""
1050         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1051             contentTempFile = self.generateTempFile(contents)
1052             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1053             if pointer_git_mode:
1054                 git_mode = pointer_git_mode
1055             if localLargeFile:
1056                 # Move temp file to final location in large file system
1057                 largeFileDir = os.path.dirname(localLargeFile)
1058                 if not os.path.isdir(largeFileDir):
1059                     os.makedirs(largeFileDir)
1060                 shutil.move(contentTempFile, localLargeFile)
1061                 self.addLargeFile(relPath)
1062                 if gitConfigBool('git-p4.largeFilePush'):
1063                     self.pushFile(localLargeFile)
1064                 if verbose:
1065                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1066         return (git_mode, contents)
1067
1068 class MockLFS(LargeFileSystem):
1069     """Mock large file system for testing."""
1070
1071     def generatePointer(self, contentFile):
1072         """The pointer content is the original content prefixed with "pointer-".
1073            The local filename of the large file storage is derived from the file content.
1074            """
1075         with open(contentFile, 'r') as f:
1076             content = next(f)
1077             gitMode = '100644'
1078             pointerContents = 'pointer-' + content
1079             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1080             return (gitMode, pointerContents, localLargeFile)
1081
1082     def pushFile(self, localLargeFile):
1083         """The remote filename of the large file storage is the same as the local
1084            one but in a different directory.
1085            """
1086         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1087         if not os.path.exists(remotePath):
1088             os.makedirs(remotePath)
1089         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1090
1091 class GitLFS(LargeFileSystem):
1092     """Git LFS as backend for the git-p4 large file system.
1093        See https://git-lfs.github.com/ for details."""
1094
1095     def __init__(self, *args):
1096         LargeFileSystem.__init__(self, *args)
1097         self.baseGitAttributes = []
1098
1099     def generatePointer(self, contentFile):
1100         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1101            mode and content which is stored in the Git repository instead of
1102            the actual content. Return also the new location of the actual
1103            content.
1104            """
1105         if os.path.getsize(contentFile) == 0:
1106             return (None, '', None)
1107
1108         pointerProcess = subprocess.Popen(
1109             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1110             stdout=subprocess.PIPE
1111         )
1112         pointerFile = pointerProcess.stdout.read()
1113         if pointerProcess.wait():
1114             os.remove(contentFile)
1115             die('git-lfs pointer command failed. Did you install the extension?')
1116
1117         # Git LFS removed the preamble in the output of the 'pointer' command
1118         # starting from version 1.2.0. Check for the preamble here to support
1119         # earlier versions.
1120         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1121         if pointerFile.startswith('Git LFS pointer for'):
1122             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1123
1124         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1125         localLargeFile = os.path.join(
1126             os.getcwd(),
1127             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1128             oid,
1129         )
1130         # LFS Spec states that pointer files should not have the executable bit set.
1131         gitMode = '100644'
1132         return (gitMode, pointerFile, localLargeFile)
1133
1134     def pushFile(self, localLargeFile):
1135         uploadProcess = subprocess.Popen(
1136             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1137         )
1138         if uploadProcess.wait():
1139             die('git-lfs push command failed. Did you define a remote?')
1140
1141     def generateGitAttributes(self):
1142         return (
1143             self.baseGitAttributes +
1144             [
1145                 '\n',
1146                 '#\n',
1147                 '# Git LFS (see https://git-lfs.github.com/)\n',
1148                 '#\n',
1149             ] +
1150             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1151                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1152             ] +
1153             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs diff=lfs merge=lfs -text\n'
1154                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1155             ]
1156         )
1157
1158     def addLargeFile(self, relPath):
1159         LargeFileSystem.addLargeFile(self, relPath)
1160         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1161
1162     def removeLargeFile(self, relPath):
1163         LargeFileSystem.removeLargeFile(self, relPath)
1164         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1165
1166     def processContent(self, git_mode, relPath, contents):
1167         if relPath == '.gitattributes':
1168             self.baseGitAttributes = contents
1169             return (git_mode, self.generateGitAttributes())
1170         else:
1171             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1172
1173 class Command:
1174     def __init__(self):
1175         self.usage = "usage: %prog [options]"
1176         self.needsGit = True
1177         self.verbose = False
1178
1179 class P4UserMap:
1180     def __init__(self):
1181         self.userMapFromPerforceServer = False
1182         self.myP4UserId = None
1183
1184     def p4UserId(self):
1185         if self.myP4UserId:
1186             return self.myP4UserId
1187
1188         results = p4CmdList("user -o")
1189         for r in results:
1190             if r.has_key('User'):
1191                 self.myP4UserId = r['User']
1192                 return r['User']
1193         die("Could not find your p4 user id")
1194
1195     def p4UserIsMe(self, p4User):
1196         # return True if the given p4 user is actually me
1197         me = self.p4UserId()
1198         if not p4User or p4User != me:
1199             return False
1200         else:
1201             return True
1202
1203     def getUserCacheFilename(self):
1204         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1205         return home + "/.gitp4-usercache.txt"
1206
1207     def getUserMapFromPerforceServer(self):
1208         if self.userMapFromPerforceServer:
1209             return
1210         self.users = {}
1211         self.emails = {}
1212
1213         for output in p4CmdList("users"):
1214             if not output.has_key("User"):
1215                 continue
1216             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1217             self.emails[output["Email"]] = output["User"]
1218
1219         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1220         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1221             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1222             if mapUser and len(mapUser[0]) == 3:
1223                 user = mapUser[0][0]
1224                 fullname = mapUser[0][1]
1225                 email = mapUser[0][2]
1226                 self.users[user] = fullname + " <" + email + ">"
1227                 self.emails[email] = user
1228
1229         s = ''
1230         for (key, val) in self.users.items():
1231             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1232
1233         open(self.getUserCacheFilename(), "wb").write(s)
1234         self.userMapFromPerforceServer = True
1235
1236     def loadUserMapFromCache(self):
1237         self.users = {}
1238         self.userMapFromPerforceServer = False
1239         try:
1240             cache = open(self.getUserCacheFilename(), "rb")
1241             lines = cache.readlines()
1242             cache.close()
1243             for line in lines:
1244                 entry = line.strip().split("\t")
1245                 self.users[entry[0]] = entry[1]
1246         except IOError:
1247             self.getUserMapFromPerforceServer()
1248
1249 class P4Debug(Command):
1250     def __init__(self):
1251         Command.__init__(self)
1252         self.options = []
1253         self.description = "A tool to debug the output of p4 -G."
1254         self.needsGit = False
1255
1256     def run(self, args):
1257         j = 0
1258         for output in p4CmdList(args):
1259             print 'Element: %d' % j
1260             j += 1
1261             print output
1262         return True
1263
1264 class P4RollBack(Command):
1265     def __init__(self):
1266         Command.__init__(self)
1267         self.options = [
1268             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1269         ]
1270         self.description = "A tool to debug the multi-branch import. Don't use :)"
1271         self.rollbackLocalBranches = False
1272
1273     def run(self, args):
1274         if len(args) != 1:
1275             return False
1276         maxChange = int(args[0])
1277
1278         if "p4ExitCode" in p4Cmd("changes -m 1"):
1279             die("Problems executing p4");
1280
1281         if self.rollbackLocalBranches:
1282             refPrefix = "refs/heads/"
1283             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1284         else:
1285             refPrefix = "refs/remotes/"
1286             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1287
1288         for line in lines:
1289             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1290                 line = line.strip()
1291                 ref = refPrefix + line
1292                 log = extractLogMessageFromGitCommit(ref)
1293                 settings = extractSettingsGitLog(log)
1294
1295                 depotPaths = settings['depot-paths']
1296                 change = settings['change']
1297
1298                 changed = False
1299
1300                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1301                                                            for p in depotPaths]))) == 0:
1302                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1303                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1304                     continue
1305
1306                 while change and int(change) > maxChange:
1307                     changed = True
1308                     if self.verbose:
1309                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1310                     system("git update-ref %s \"%s^\"" % (ref, ref))
1311                     log = extractLogMessageFromGitCommit(ref)
1312                     settings =  extractSettingsGitLog(log)
1313
1314
1315                     depotPaths = settings['depot-paths']
1316                     change = settings['change']
1317
1318                 if changed:
1319                     print "%s rewound to %s" % (ref, change)
1320
1321         return True
1322
1323 class P4Submit(Command, P4UserMap):
1324
1325     conflict_behavior_choices = ("ask", "skip", "quit")
1326
1327     def __init__(self):
1328         Command.__init__(self)
1329         P4UserMap.__init__(self)
1330         self.options = [
1331                 optparse.make_option("--origin", dest="origin"),
1332                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1333                 # preserve the user, requires relevant p4 permissions
1334                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1335                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1336                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1337                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1338                 optparse.make_option("--conflict", dest="conflict_behavior",
1339                                      choices=self.conflict_behavior_choices),
1340                 optparse.make_option("--branch", dest="branch"),
1341                 optparse.make_option("--shelve", dest="shelve", action="store_true",
1342                                      help="Shelve instead of submit. Shelved files are reverted, "
1343                                      "restoring the workspace to the state before the shelve"),
1344                 optparse.make_option("--update-shelve", dest="update_shelve", action="store", type="int",
1345                                      metavar="CHANGELIST",
1346                                      help="update an existing shelved changelist, implies --shelve")
1347         ]
1348         self.description = "Submit changes from git to the perforce depot."
1349         self.usage += " [name of git branch to submit into perforce depot]"
1350         self.origin = ""
1351         self.detectRenames = False
1352         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1353         self.dry_run = False
1354         self.shelve = False
1355         self.update_shelve = None
1356         self.prepare_p4_only = False
1357         self.conflict_behavior = None
1358         self.isWindows = (platform.system() == "Windows")
1359         self.exportLabels = False
1360         self.p4HasMoveCommand = p4_has_move_command()
1361         self.branch = None
1362
1363         if gitConfig('git-p4.largeFileSystem'):
1364             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1365
1366     def check(self):
1367         if len(p4CmdList("opened ...")) > 0:
1368             die("You have files opened with perforce! Close them before starting the sync.")
1369
1370     def separate_jobs_from_description(self, message):
1371         """Extract and return a possible Jobs field in the commit
1372            message.  It goes into a separate section in the p4 change
1373            specification.
1374
1375            A jobs line starts with "Jobs:" and looks like a new field
1376            in a form.  Values are white-space separated on the same
1377            line or on following lines that start with a tab.
1378
1379            This does not parse and extract the full git commit message
1380            like a p4 form.  It just sees the Jobs: line as a marker
1381            to pass everything from then on directly into the p4 form,
1382            but outside the description section.
1383
1384            Return a tuple (stripped log message, jobs string)."""
1385
1386         m = re.search(r'^Jobs:', message, re.MULTILINE)
1387         if m is None:
1388             return (message, None)
1389
1390         jobtext = message[m.start():]
1391         stripped_message = message[:m.start()].rstrip()
1392         return (stripped_message, jobtext)
1393
1394     def prepareLogMessage(self, template, message, jobs):
1395         """Edits the template returned from "p4 change -o" to insert
1396            the message in the Description field, and the jobs text in
1397            the Jobs field."""
1398         result = ""
1399
1400         inDescriptionSection = False
1401
1402         for line in template.split("\n"):
1403             if line.startswith("#"):
1404                 result += line + "\n"
1405                 continue
1406
1407             if inDescriptionSection:
1408                 if line.startswith("Files:") or line.startswith("Jobs:"):
1409                     inDescriptionSection = False
1410                     # insert Jobs section
1411                     if jobs:
1412                         result += jobs + "\n"
1413                 else:
1414                     continue
1415             else:
1416                 if line.startswith("Description:"):
1417                     inDescriptionSection = True
1418                     line += "\n"
1419                     for messageLine in message.split("\n"):
1420                         line += "\t" + messageLine + "\n"
1421
1422             result += line + "\n"
1423
1424         return result
1425
1426     def patchRCSKeywords(self, file, pattern):
1427         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1428         (handle, outFileName) = tempfile.mkstemp(dir='.')
1429         try:
1430             outFile = os.fdopen(handle, "w+")
1431             inFile = open(file, "r")
1432             regexp = re.compile(pattern, re.VERBOSE)
1433             for line in inFile.readlines():
1434                 line = regexp.sub(r'$\1$', line)
1435                 outFile.write(line)
1436             inFile.close()
1437             outFile.close()
1438             # Forcibly overwrite the original file
1439             os.unlink(file)
1440             shutil.move(outFileName, file)
1441         except:
1442             # cleanup our temporary file
1443             os.unlink(outFileName)
1444             print "Failed to strip RCS keywords in %s" % file
1445             raise
1446
1447         print "Patched up RCS keywords in %s" % file
1448
1449     def p4UserForCommit(self,id):
1450         # Return the tuple (perforce user,git email) for a given git commit id
1451         self.getUserMapFromPerforceServer()
1452         gitEmail = read_pipe(["git", "log", "--max-count=1",
1453                               "--format=%ae", id])
1454         gitEmail = gitEmail.strip()
1455         if not self.emails.has_key(gitEmail):
1456             return (None,gitEmail)
1457         else:
1458             return (self.emails[gitEmail],gitEmail)
1459
1460     def checkValidP4Users(self,commits):
1461         # check if any git authors cannot be mapped to p4 users
1462         for id in commits:
1463             (user,email) = self.p4UserForCommit(id)
1464             if not user:
1465                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1466                 if gitConfigBool("git-p4.allowMissingP4Users"):
1467                     print "%s" % msg
1468                 else:
1469                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1470
1471     def lastP4Changelist(self):
1472         # Get back the last changelist number submitted in this client spec. This
1473         # then gets used to patch up the username in the change. If the same
1474         # client spec is being used by multiple processes then this might go
1475         # wrong.
1476         results = p4CmdList("client -o")        # find the current client
1477         client = None
1478         for r in results:
1479             if r.has_key('Client'):
1480                 client = r['Client']
1481                 break
1482         if not client:
1483             die("could not get client spec")
1484         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1485         for r in results:
1486             if r.has_key('change'):
1487                 return r['change']
1488         die("Could not get changelist number for last submit - cannot patch up user details")
1489
1490     def modifyChangelistUser(self, changelist, newUser):
1491         # fixup the user field of a changelist after it has been submitted.
1492         changes = p4CmdList("change -o %s" % changelist)
1493         if len(changes) != 1:
1494             die("Bad output from p4 change modifying %s to user %s" %
1495                 (changelist, newUser))
1496
1497         c = changes[0]
1498         if c['User'] == newUser: return   # nothing to do
1499         c['User'] = newUser
1500         input = marshal.dumps(c)
1501
1502         result = p4CmdList("change -f -i", stdin=input)
1503         for r in result:
1504             if r.has_key('code'):
1505                 if r['code'] == 'error':
1506                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1507             if r.has_key('data'):
1508                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1509                 return
1510         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1511
1512     def canChangeChangelists(self):
1513         # check to see if we have p4 admin or super-user permissions, either of
1514         # which are required to modify changelists.
1515         results = p4CmdList(["protects", self.depotPath])
1516         for r in results:
1517             if r.has_key('perm'):
1518                 if r['perm'] == 'admin':
1519                     return 1
1520                 if r['perm'] == 'super':
1521                     return 1
1522         return 0
1523
1524     def prepareSubmitTemplate(self, changelist=None):
1525         """Run "p4 change -o" to grab a change specification template.
1526            This does not use "p4 -G", as it is nice to keep the submission
1527            template in original order, since a human might edit it.
1528
1529            Remove lines in the Files section that show changes to files
1530            outside the depot path we're committing into."""
1531
1532         [upstream, settings] = findUpstreamBranchPoint()
1533
1534         template = ""
1535         inFilesSection = False
1536         args = ['change', '-o']
1537         if changelist:
1538             args.append(str(changelist))
1539
1540         for line in p4_read_pipe_lines(args):
1541             if line.endswith("\r\n"):
1542                 line = line[:-2] + "\n"
1543             if inFilesSection:
1544                 if line.startswith("\t"):
1545                     # path starts and ends with a tab
1546                     path = line[1:]
1547                     lastTab = path.rfind("\t")
1548                     if lastTab != -1:
1549                         path = path[:lastTab]
1550                         if settings.has_key('depot-paths'):
1551                             if not [p for p in settings['depot-paths']
1552                                     if p4PathStartsWith(path, p)]:
1553                                 continue
1554                         else:
1555                             if not p4PathStartsWith(path, self.depotPath):
1556                                 continue
1557                 else:
1558                     inFilesSection = False
1559             else:
1560                 if line.startswith("Files:"):
1561                     inFilesSection = True
1562
1563             template += line
1564
1565         return template
1566
1567     def edit_template(self, template_file):
1568         """Invoke the editor to let the user change the submission
1569            message.  Return true if okay to continue with the submit."""
1570
1571         # if configured to skip the editing part, just submit
1572         if gitConfigBool("git-p4.skipSubmitEdit"):
1573             return True
1574
1575         # look at the modification time, to check later if the user saved
1576         # the file
1577         mtime = os.stat(template_file).st_mtime
1578
1579         # invoke the editor
1580         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1581             editor = os.environ.get("P4EDITOR")
1582         else:
1583             editor = read_pipe("git var GIT_EDITOR").strip()
1584         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1585
1586         # If the file was not saved, prompt to see if this patch should
1587         # be skipped.  But skip this verification step if configured so.
1588         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1589             return True
1590
1591         # modification time updated means user saved the file
1592         if os.stat(template_file).st_mtime > mtime:
1593             return True
1594
1595         while True:
1596             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1597             if response == 'y':
1598                 return True
1599             if response == 'n':
1600                 return False
1601
1602     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1603         # diff
1604         if os.environ.has_key("P4DIFF"):
1605             del(os.environ["P4DIFF"])
1606         diff = ""
1607         for editedFile in editedFiles:
1608             diff += p4_read_pipe(['diff', '-du',
1609                                   wildcard_encode(editedFile)])
1610
1611         # new file diff
1612         newdiff = ""
1613         for newFile in filesToAdd:
1614             newdiff += "==== new file ====\n"
1615             newdiff += "--- /dev/null\n"
1616             newdiff += "+++ %s\n" % newFile
1617
1618             is_link = os.path.islink(newFile)
1619             expect_link = newFile in symlinks
1620
1621             if is_link and expect_link:
1622                 newdiff += "+%s\n" % os.readlink(newFile)
1623             else:
1624                 f = open(newFile, "r")
1625                 for line in f.readlines():
1626                     newdiff += "+" + line
1627                 f.close()
1628
1629         return (diff + newdiff).replace('\r\n', '\n')
1630
1631     def applyCommit(self, id):
1632         """Apply one commit, return True if it succeeded."""
1633
1634         print "Applying", read_pipe(["git", "show", "-s",
1635                                      "--format=format:%h %s", id])
1636
1637         (p4User, gitEmail) = self.p4UserForCommit(id)
1638
1639         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1640         filesToAdd = set()
1641         filesToChangeType = set()
1642         filesToDelete = set()
1643         editedFiles = set()
1644         pureRenameCopy = set()
1645         symlinks = set()
1646         filesToChangeExecBit = {}
1647         all_files = list()
1648
1649         for line in diff:
1650             diff = parseDiffTreeEntry(line)
1651             modifier = diff['status']
1652             path = diff['src']
1653             all_files.append(path)
1654
1655             if modifier == "M":
1656                 p4_edit(path)
1657                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1658                     filesToChangeExecBit[path] = diff['dst_mode']
1659                 editedFiles.add(path)
1660             elif modifier == "A":
1661                 filesToAdd.add(path)
1662                 filesToChangeExecBit[path] = diff['dst_mode']
1663                 if path in filesToDelete:
1664                     filesToDelete.remove(path)
1665
1666                 dst_mode = int(diff['dst_mode'], 8)
1667                 if dst_mode == 0120000:
1668                     symlinks.add(path)
1669
1670             elif modifier == "D":
1671                 filesToDelete.add(path)
1672                 if path in filesToAdd:
1673                     filesToAdd.remove(path)
1674             elif modifier == "C":
1675                 src, dest = diff['src'], diff['dst']
1676                 p4_integrate(src, dest)
1677                 pureRenameCopy.add(dest)
1678                 if diff['src_sha1'] != diff['dst_sha1']:
1679                     p4_edit(dest)
1680                     pureRenameCopy.discard(dest)
1681                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1682                     p4_edit(dest)
1683                     pureRenameCopy.discard(dest)
1684                     filesToChangeExecBit[dest] = diff['dst_mode']
1685                 if self.isWindows:
1686                     # turn off read-only attribute
1687                     os.chmod(dest, stat.S_IWRITE)
1688                 os.unlink(dest)
1689                 editedFiles.add(dest)
1690             elif modifier == "R":
1691                 src, dest = diff['src'], diff['dst']
1692                 if self.p4HasMoveCommand:
1693                     p4_edit(src)        # src must be open before move
1694                     p4_move(src, dest)  # opens for (move/delete, move/add)
1695                 else:
1696                     p4_integrate(src, dest)
1697                     if diff['src_sha1'] != diff['dst_sha1']:
1698                         p4_edit(dest)
1699                     else:
1700                         pureRenameCopy.add(dest)
1701                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1702                     if not self.p4HasMoveCommand:
1703                         p4_edit(dest)   # with move: already open, writable
1704                     filesToChangeExecBit[dest] = diff['dst_mode']
1705                 if not self.p4HasMoveCommand:
1706                     if self.isWindows:
1707                         os.chmod(dest, stat.S_IWRITE)
1708                     os.unlink(dest)
1709                     filesToDelete.add(src)
1710                 editedFiles.add(dest)
1711             elif modifier == "T":
1712                 filesToChangeType.add(path)
1713             else:
1714                 die("unknown modifier %s for %s" % (modifier, path))
1715
1716         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1717         patchcmd = diffcmd + " | git apply "
1718         tryPatchCmd = patchcmd + "--check -"
1719         applyPatchCmd = patchcmd + "--check --apply -"
1720         patch_succeeded = True
1721
1722         if os.system(tryPatchCmd) != 0:
1723             fixed_rcs_keywords = False
1724             patch_succeeded = False
1725             print "Unfortunately applying the change failed!"
1726
1727             # Patch failed, maybe it's just RCS keyword woes. Look through
1728             # the patch to see if that's possible.
1729             if gitConfigBool("git-p4.attemptRCSCleanup"):
1730                 file = None
1731                 pattern = None
1732                 kwfiles = {}
1733                 for file in editedFiles | filesToDelete:
1734                     # did this file's delta contain RCS keywords?
1735                     pattern = p4_keywords_regexp_for_file(file)
1736
1737                     if pattern:
1738                         # this file is a possibility...look for RCS keywords.
1739                         regexp = re.compile(pattern, re.VERBOSE)
1740                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1741                             if regexp.search(line):
1742                                 if verbose:
1743                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1744                                 kwfiles[file] = pattern
1745                                 break
1746
1747                 for file in kwfiles:
1748                     if verbose:
1749                         print "zapping %s with %s" % (line,pattern)
1750                     # File is being deleted, so not open in p4.  Must
1751                     # disable the read-only bit on windows.
1752                     if self.isWindows and file not in editedFiles:
1753                         os.chmod(file, stat.S_IWRITE)
1754                     self.patchRCSKeywords(file, kwfiles[file])
1755                     fixed_rcs_keywords = True
1756
1757             if fixed_rcs_keywords:
1758                 print "Retrying the patch with RCS keywords cleaned up"
1759                 if os.system(tryPatchCmd) == 0:
1760                     patch_succeeded = True
1761
1762         if not patch_succeeded:
1763             for f in editedFiles:
1764                 p4_revert(f)
1765             return False
1766
1767         #
1768         # Apply the patch for real, and do add/delete/+x handling.
1769         #
1770         system(applyPatchCmd)
1771
1772         for f in filesToChangeType:
1773             p4_edit(f, "-t", "auto")
1774         for f in filesToAdd:
1775             p4_add(f)
1776         for f in filesToDelete:
1777             p4_revert(f)
1778             p4_delete(f)
1779
1780         # Set/clear executable bits
1781         for f in filesToChangeExecBit.keys():
1782             mode = filesToChangeExecBit[f]
1783             setP4ExecBit(f, mode)
1784
1785         if self.update_shelve:
1786             print("all_files = %s" % str(all_files))
1787             p4_reopen_in_change(self.update_shelve, all_files)
1788
1789         #
1790         # Build p4 change description, starting with the contents
1791         # of the git commit message.
1792         #
1793         logMessage = extractLogMessageFromGitCommit(id)
1794         logMessage = logMessage.strip()
1795         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1796
1797         template = self.prepareSubmitTemplate(self.update_shelve)
1798         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1799
1800         if self.preserveUser:
1801            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1802
1803         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1804             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1805             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1806             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1807
1808         separatorLine = "######## everything below this line is just the diff #######\n"
1809         if not self.prepare_p4_only:
1810             submitTemplate += separatorLine
1811             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1812
1813         (handle, fileName) = tempfile.mkstemp()
1814         tmpFile = os.fdopen(handle, "w+b")
1815         if self.isWindows:
1816             submitTemplate = submitTemplate.replace("\n", "\r\n")
1817         tmpFile.write(submitTemplate)
1818         tmpFile.close()
1819
1820         if self.prepare_p4_only:
1821             #
1822             # Leave the p4 tree prepared, and the submit template around
1823             # and let the user decide what to do next
1824             #
1825             print
1826             print "P4 workspace prepared for submission."
1827             print "To submit or revert, go to client workspace"
1828             print "  " + self.clientPath
1829             print
1830             print "To submit, use \"p4 submit\" to write a new description,"
1831             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1832                   " \"git p4\"." % fileName
1833             print "You can delete the file \"%s\" when finished." % fileName
1834
1835             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1836                 print "To preserve change ownership by user %s, you must\n" \
1837                       "do \"p4 change -f <change>\" after submitting and\n" \
1838                       "edit the User field."
1839             if pureRenameCopy:
1840                 print "After submitting, renamed files must be re-synced."
1841                 print "Invoke \"p4 sync -f\" on each of these files:"
1842                 for f in pureRenameCopy:
1843                     print "  " + f
1844
1845             print
1846             print "To revert the changes, use \"p4 revert ...\", and delete"
1847             print "the submit template file \"%s\"" % fileName
1848             if filesToAdd:
1849                 print "Since the commit adds new files, they must be deleted:"
1850                 for f in filesToAdd:
1851                     print "  " + f
1852             print
1853             return True
1854
1855         #
1856         # Let the user edit the change description, then submit it.
1857         #
1858         submitted = False
1859
1860         try:
1861             if self.edit_template(fileName):
1862                 # read the edited message and submit
1863                 tmpFile = open(fileName, "rb")
1864                 message = tmpFile.read()
1865                 tmpFile.close()
1866                 if self.isWindows:
1867                     message = message.replace("\r\n", "\n")
1868                 submitTemplate = message[:message.index(separatorLine)]
1869
1870                 if self.update_shelve:
1871                     p4_write_pipe(['shelve', '-r', '-i'], submitTemplate)
1872                 elif self.shelve:
1873                     p4_write_pipe(['shelve', '-i'], submitTemplate)
1874                 else:
1875                     p4_write_pipe(['submit', '-i'], submitTemplate)
1876                     # The rename/copy happened by applying a patch that created a
1877                     # new file.  This leaves it writable, which confuses p4.
1878                     for f in pureRenameCopy:
1879                         p4_sync(f, "-f")
1880
1881                 if self.preserveUser:
1882                     if p4User:
1883                         # Get last changelist number. Cannot easily get it from
1884                         # the submit command output as the output is
1885                         # unmarshalled.
1886                         changelist = self.lastP4Changelist()
1887                         self.modifyChangelistUser(changelist, p4User)
1888
1889                 submitted = True
1890
1891         finally:
1892             # skip this patch
1893             if not submitted or self.shelve:
1894                 if self.shelve:
1895                     print ("Reverting shelved files.")
1896                 else:
1897                     print ("Submission cancelled, undoing p4 changes.")
1898                 for f in editedFiles | filesToDelete:
1899                     p4_revert(f)
1900                 for f in filesToAdd:
1901                     p4_revert(f)
1902                     os.remove(f)
1903
1904         os.remove(fileName)
1905         return submitted
1906
1907     # Export git tags as p4 labels. Create a p4 label and then tag
1908     # with that.
1909     def exportGitTags(self, gitTags):
1910         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1911         if len(validLabelRegexp) == 0:
1912             validLabelRegexp = defaultLabelRegexp
1913         m = re.compile(validLabelRegexp)
1914
1915         for name in gitTags:
1916
1917             if not m.match(name):
1918                 if verbose:
1919                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1920                 continue
1921
1922             # Get the p4 commit this corresponds to
1923             logMessage = extractLogMessageFromGitCommit(name)
1924             values = extractSettingsGitLog(logMessage)
1925
1926             if not values.has_key('change'):
1927                 # a tag pointing to something not sent to p4; ignore
1928                 if verbose:
1929                     print "git tag %s does not give a p4 commit" % name
1930                 continue
1931             else:
1932                 changelist = values['change']
1933
1934             # Get the tag details.
1935             inHeader = True
1936             isAnnotated = False
1937             body = []
1938             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1939                 l = l.strip()
1940                 if inHeader:
1941                     if re.match(r'tag\s+', l):
1942                         isAnnotated = True
1943                     elif re.match(r'\s*$', l):
1944                         inHeader = False
1945                         continue
1946                 else:
1947                     body.append(l)
1948
1949             if not isAnnotated:
1950                 body = ["lightweight tag imported by git p4\n"]
1951
1952             # Create the label - use the same view as the client spec we are using
1953             clientSpec = getClientSpec()
1954
1955             labelTemplate  = "Label: %s\n" % name
1956             labelTemplate += "Description:\n"
1957             for b in body:
1958                 labelTemplate += "\t" + b + "\n"
1959             labelTemplate += "View:\n"
1960             for depot_side in clientSpec.mappings:
1961                 labelTemplate += "\t%s\n" % depot_side
1962
1963             if self.dry_run:
1964                 print "Would create p4 label %s for tag" % name
1965             elif self.prepare_p4_only:
1966                 print "Not creating p4 label %s for tag due to option" \
1967                       " --prepare-p4-only" % name
1968             else:
1969                 p4_write_pipe(["label", "-i"], labelTemplate)
1970
1971                 # Use the label
1972                 p4_system(["tag", "-l", name] +
1973                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1974
1975                 if verbose:
1976                     print "created p4 label for tag %s" % name
1977
1978     def run(self, args):
1979         if len(args) == 0:
1980             self.master = currentGitBranch()
1981         elif len(args) == 1:
1982             self.master = args[0]
1983             if not branchExists(self.master):
1984                 die("Branch %s does not exist" % self.master)
1985         else:
1986             return False
1987
1988         if self.master:
1989             allowSubmit = gitConfig("git-p4.allowSubmit")
1990             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1991                 die("%s is not in git-p4.allowSubmit" % self.master)
1992
1993         [upstream, settings] = findUpstreamBranchPoint()
1994         self.depotPath = settings['depot-paths'][0]
1995         if len(self.origin) == 0:
1996             self.origin = upstream
1997
1998         if self.update_shelve:
1999             self.shelve = True
2000
2001         if self.preserveUser:
2002             if not self.canChangeChangelists():
2003                 die("Cannot preserve user names without p4 super-user or admin permissions")
2004
2005         # if not set from the command line, try the config file
2006         if self.conflict_behavior is None:
2007             val = gitConfig("git-p4.conflict")
2008             if val:
2009                 if val not in self.conflict_behavior_choices:
2010                     die("Invalid value '%s' for config git-p4.conflict" % val)
2011             else:
2012                 val = "ask"
2013             self.conflict_behavior = val
2014
2015         if self.verbose:
2016             print "Origin branch is " + self.origin
2017
2018         if len(self.depotPath) == 0:
2019             print "Internal error: cannot locate perforce depot path from existing branches"
2020             sys.exit(128)
2021
2022         self.useClientSpec = False
2023         if gitConfigBool("git-p4.useclientspec"):
2024             self.useClientSpec = True
2025         if self.useClientSpec:
2026             self.clientSpecDirs = getClientSpec()
2027
2028         # Check for the existence of P4 branches
2029         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
2030
2031         if self.useClientSpec and not branchesDetected:
2032             # all files are relative to the client spec
2033             self.clientPath = getClientRoot()
2034         else:
2035             self.clientPath = p4Where(self.depotPath)
2036
2037         if self.clientPath == "":
2038             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
2039
2040         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
2041         self.oldWorkingDirectory = os.getcwd()
2042
2043         # ensure the clientPath exists
2044         new_client_dir = False
2045         if not os.path.exists(self.clientPath):
2046             new_client_dir = True
2047             os.makedirs(self.clientPath)
2048
2049         chdir(self.clientPath, is_client_path=True)
2050         if self.dry_run:
2051             print "Would synchronize p4 checkout in %s" % self.clientPath
2052         else:
2053             print "Synchronizing p4 checkout..."
2054             if new_client_dir:
2055                 # old one was destroyed, and maybe nobody told p4
2056                 p4_sync("...", "-f")
2057             else:
2058                 p4_sync("...")
2059         self.check()
2060
2061         commits = []
2062         if self.master:
2063             commitish = self.master
2064         else:
2065             commitish = 'HEAD'
2066
2067         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
2068             commits.append(line.strip())
2069         commits.reverse()
2070
2071         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2072             self.checkAuthorship = False
2073         else:
2074             self.checkAuthorship = True
2075
2076         if self.preserveUser:
2077             self.checkValidP4Users(commits)
2078
2079         #
2080         # Build up a set of options to be passed to diff when
2081         # submitting each commit to p4.
2082         #
2083         if self.detectRenames:
2084             # command-line -M arg
2085             self.diffOpts = "-M"
2086         else:
2087             # If not explicitly set check the config variable
2088             detectRenames = gitConfig("git-p4.detectRenames")
2089
2090             if detectRenames.lower() == "false" or detectRenames == "":
2091                 self.diffOpts = ""
2092             elif detectRenames.lower() == "true":
2093                 self.diffOpts = "-M"
2094             else:
2095                 self.diffOpts = "-M%s" % detectRenames
2096
2097         # no command-line arg for -C or --find-copies-harder, just
2098         # config variables
2099         detectCopies = gitConfig("git-p4.detectCopies")
2100         if detectCopies.lower() == "false" or detectCopies == "":
2101             pass
2102         elif detectCopies.lower() == "true":
2103             self.diffOpts += " -C"
2104         else:
2105             self.diffOpts += " -C%s" % detectCopies
2106
2107         if gitConfigBool("git-p4.detectCopiesHarder"):
2108             self.diffOpts += " --find-copies-harder"
2109
2110         #
2111         # Apply the commits, one at a time.  On failure, ask if should
2112         # continue to try the rest of the patches, or quit.
2113         #
2114         if self.dry_run:
2115             print "Would apply"
2116         applied = []
2117         last = len(commits) - 1
2118         for i, commit in enumerate(commits):
2119             if self.dry_run:
2120                 print " ", read_pipe(["git", "show", "-s",
2121                                       "--format=format:%h %s", commit])
2122                 ok = True
2123             else:
2124                 ok = self.applyCommit(commit)
2125             if ok:
2126                 applied.append(commit)
2127             else:
2128                 if self.prepare_p4_only and i < last:
2129                     print "Processing only the first commit due to option" \
2130                           " --prepare-p4-only"
2131                     break
2132                 if i < last:
2133                     quit = False
2134                     while True:
2135                         # prompt for what to do, or use the option/variable
2136                         if self.conflict_behavior == "ask":
2137                             print "What do you want to do?"
2138                             response = raw_input("[s]kip this commit but apply"
2139                                                  " the rest, or [q]uit? ")
2140                             if not response:
2141                                 continue
2142                         elif self.conflict_behavior == "skip":
2143                             response = "s"
2144                         elif self.conflict_behavior == "quit":
2145                             response = "q"
2146                         else:
2147                             die("Unknown conflict_behavior '%s'" %
2148                                 self.conflict_behavior)
2149
2150                         if response[0] == "s":
2151                             print "Skipping this commit, but applying the rest"
2152                             break
2153                         if response[0] == "q":
2154                             print "Quitting"
2155                             quit = True
2156                             break
2157                     if quit:
2158                         break
2159
2160         chdir(self.oldWorkingDirectory)
2161         shelved_applied = "shelved" if self.shelve else "applied"
2162         if self.dry_run:
2163             pass
2164         elif self.prepare_p4_only:
2165             pass
2166         elif len(commits) == len(applied):
2167             print ("All commits {0}!".format(shelved_applied))
2168
2169             sync = P4Sync()
2170             if self.branch:
2171                 sync.branch = self.branch
2172             sync.run([])
2173
2174             rebase = P4Rebase()
2175             rebase.rebase()
2176
2177         else:
2178             if len(applied) == 0:
2179                 print ("No commits {0}.".format(shelved_applied))
2180             else:
2181                 print ("{0} only the commits marked with '*':".format(shelved_applied.capitalize()))
2182                 for c in commits:
2183                     if c in applied:
2184                         star = "*"
2185                     else:
2186                         star = " "
2187                     print star, read_pipe(["git", "show", "-s",
2188                                            "--format=format:%h %s",  c])
2189                 print "You will have to do 'git p4 sync' and rebase."
2190
2191         if gitConfigBool("git-p4.exportLabels"):
2192             self.exportLabels = True
2193
2194         if self.exportLabels:
2195             p4Labels = getP4Labels(self.depotPath)
2196             gitTags = getGitTags()
2197
2198             missingGitTags = gitTags - p4Labels
2199             self.exportGitTags(missingGitTags)
2200
2201         # exit with error unless everything applied perfectly
2202         if len(commits) != len(applied):
2203                 sys.exit(1)
2204
2205         return True
2206
2207 class View(object):
2208     """Represent a p4 view ("p4 help views"), and map files in a
2209        repo according to the view."""
2210
2211     def __init__(self, client_name):
2212         self.mappings = []
2213         self.client_prefix = "//%s/" % client_name
2214         # cache results of "p4 where" to lookup client file locations
2215         self.client_spec_path_cache = {}
2216
2217     def append(self, view_line):
2218         """Parse a view line, splitting it into depot and client
2219            sides.  Append to self.mappings, preserving order.  This
2220            is only needed for tag creation."""
2221
2222         # Split the view line into exactly two words.  P4 enforces
2223         # structure on these lines that simplifies this quite a bit.
2224         #
2225         # Either or both words may be double-quoted.
2226         # Single quotes do not matter.
2227         # Double-quote marks cannot occur inside the words.
2228         # A + or - prefix is also inside the quotes.
2229         # There are no quotes unless they contain a space.
2230         # The line is already white-space stripped.
2231         # The two words are separated by a single space.
2232         #
2233         if view_line[0] == '"':
2234             # First word is double quoted.  Find its end.
2235             close_quote_index = view_line.find('"', 1)
2236             if close_quote_index <= 0:
2237                 die("No first-word closing quote found: %s" % view_line)
2238             depot_side = view_line[1:close_quote_index]
2239             # skip closing quote and space
2240             rhs_index = close_quote_index + 1 + 1
2241         else:
2242             space_index = view_line.find(" ")
2243             if space_index <= 0:
2244                 die("No word-splitting space found: %s" % view_line)
2245             depot_side = view_line[0:space_index]
2246             rhs_index = space_index + 1
2247
2248         # prefix + means overlay on previous mapping
2249         if depot_side.startswith("+"):
2250             depot_side = depot_side[1:]
2251
2252         # prefix - means exclude this path, leave out of mappings
2253         exclude = False
2254         if depot_side.startswith("-"):
2255             exclude = True
2256             depot_side = depot_side[1:]
2257
2258         if not exclude:
2259             self.mappings.append(depot_side)
2260
2261     def convert_client_path(self, clientFile):
2262         # chop off //client/ part to make it relative
2263         if not clientFile.startswith(self.client_prefix):
2264             die("No prefix '%s' on clientFile '%s'" %
2265                 (self.client_prefix, clientFile))
2266         return clientFile[len(self.client_prefix):]
2267
2268     def update_client_spec_path_cache(self, files):
2269         """ Caching file paths by "p4 where" batch query """
2270
2271         # List depot file paths exclude that already cached
2272         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2273
2274         if len(fileArgs) == 0:
2275             return  # All files in cache
2276
2277         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2278         for res in where_result:
2279             if "code" in res and res["code"] == "error":
2280                 # assume error is "... file(s) not in client view"
2281                 continue
2282             if "clientFile" not in res:
2283                 die("No clientFile in 'p4 where' output")
2284             if "unmap" in res:
2285                 # it will list all of them, but only one not unmap-ped
2286                 continue
2287             if gitConfigBool("core.ignorecase"):
2288                 res['depotFile'] = res['depotFile'].lower()
2289             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2290
2291         # not found files or unmap files set to ""
2292         for depotFile in fileArgs:
2293             if gitConfigBool("core.ignorecase"):
2294                 depotFile = depotFile.lower()
2295             if depotFile not in self.client_spec_path_cache:
2296                 self.client_spec_path_cache[depotFile] = ""
2297
2298     def map_in_client(self, depot_path):
2299         """Return the relative location in the client where this
2300            depot file should live.  Returns "" if the file should
2301            not be mapped in the client."""
2302
2303         if gitConfigBool("core.ignorecase"):
2304             depot_path = depot_path.lower()
2305
2306         if depot_path in self.client_spec_path_cache:
2307             return self.client_spec_path_cache[depot_path]
2308
2309         die( "Error: %s is not found in client spec path" % depot_path )
2310         return ""
2311
2312 class P4Sync(Command, P4UserMap):
2313     delete_actions = ( "delete", "move/delete", "purge" )
2314
2315     def __init__(self):
2316         Command.__init__(self)
2317         P4UserMap.__init__(self)
2318         self.options = [
2319                 optparse.make_option("--branch", dest="branch"),
2320                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2321                 optparse.make_option("--changesfile", dest="changesFile"),
2322                 optparse.make_option("--silent", dest="silent", action="store_true"),
2323                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2324                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2325                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2326                                      help="Import into refs/heads/ , not refs/remotes"),
2327                 optparse.make_option("--max-changes", dest="maxChanges",
2328                                      help="Maximum number of changes to import"),
2329                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2330                                      help="Internal block size to use when iteratively calling p4 changes"),
2331                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2332                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2333                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2334                                      help="Only sync files that are included in the Perforce Client Spec"),
2335                 optparse.make_option("-/", dest="cloneExclude",
2336                                      action="append", type="string",
2337                                      help="exclude depot path"),
2338         ]
2339         self.description = """Imports from Perforce into a git repository.\n
2340     example:
2341     //depot/my/project/ -- to import the current head
2342     //depot/my/project/@all -- to import everything
2343     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2344
2345     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2346
2347         self.usage += " //depot/path[@revRange]"
2348         self.silent = False
2349         self.createdBranches = set()
2350         self.committedChanges = set()
2351         self.branch = ""
2352         self.detectBranches = False
2353         self.detectLabels = False
2354         self.importLabels = False
2355         self.changesFile = ""
2356         self.syncWithOrigin = True
2357         self.importIntoRemotes = True
2358         self.maxChanges = ""
2359         self.changes_block_size = None
2360         self.keepRepoPath = False
2361         self.depotPaths = None
2362         self.p4BranchesInGit = []
2363         self.cloneExclude = []
2364         self.useClientSpec = False
2365         self.useClientSpec_from_options = False
2366         self.clientSpecDirs = None
2367         self.tempBranches = []
2368         self.tempBranchLocation = "refs/git-p4-tmp"
2369         self.largeFileSystem = None
2370
2371         if gitConfig('git-p4.largeFileSystem'):
2372             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2373             self.largeFileSystem = largeFileSystemConstructor(
2374                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2375             )
2376
2377         if gitConfig("git-p4.syncFromOrigin") == "false":
2378             self.syncWithOrigin = False
2379
2380     # This is required for the "append" cloneExclude action
2381     def ensure_value(self, attr, value):
2382         if not hasattr(self, attr) or getattr(self, attr) is None:
2383             setattr(self, attr, value)
2384         return getattr(self, attr)
2385
2386     # Force a checkpoint in fast-import and wait for it to finish
2387     def checkpoint(self):
2388         self.gitStream.write("checkpoint\n\n")
2389         self.gitStream.write("progress checkpoint\n\n")
2390         out = self.gitOutput.readline()
2391         if self.verbose:
2392             print "checkpoint finished: " + out
2393
2394     def extractFilesFromCommit(self, commit):
2395         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2396                              for path in self.cloneExclude]
2397         files = []
2398         fnum = 0
2399         while commit.has_key("depotFile%s" % fnum):
2400             path =  commit["depotFile%s" % fnum]
2401
2402             if [p for p in self.cloneExclude
2403                 if p4PathStartsWith(path, p)]:
2404                 found = False
2405             else:
2406                 found = [p for p in self.depotPaths
2407                          if p4PathStartsWith(path, p)]
2408             if not found:
2409                 fnum = fnum + 1
2410                 continue
2411
2412             file = {}
2413             file["path"] = path
2414             file["rev"] = commit["rev%s" % fnum]
2415             file["action"] = commit["action%s" % fnum]
2416             file["type"] = commit["type%s" % fnum]
2417             files.append(file)
2418             fnum = fnum + 1
2419         return files
2420
2421     def extractJobsFromCommit(self, commit):
2422         jobs = []
2423         jnum = 0
2424         while commit.has_key("job%s" % jnum):
2425             job = commit["job%s" % jnum]
2426             jobs.append(job)
2427             jnum = jnum + 1
2428         return jobs
2429
2430     def stripRepoPath(self, path, prefixes):
2431         """When streaming files, this is called to map a p4 depot path
2432            to where it should go in git.  The prefixes are either
2433            self.depotPaths, or self.branchPrefixes in the case of
2434            branch detection."""
2435
2436         if self.useClientSpec:
2437             # branch detection moves files up a level (the branch name)
2438             # from what client spec interpretation gives
2439             path = self.clientSpecDirs.map_in_client(path)
2440             if self.detectBranches:
2441                 for b in self.knownBranches:
2442                     if path.startswith(b + "/"):
2443                         path = path[len(b)+1:]
2444
2445         elif self.keepRepoPath:
2446             # Preserve everything in relative path name except leading
2447             # //depot/; just look at first prefix as they all should
2448             # be in the same depot.
2449             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2450             if p4PathStartsWith(path, depot):
2451                 path = path[len(depot):]
2452
2453         else:
2454             for p in prefixes:
2455                 if p4PathStartsWith(path, p):
2456                     path = path[len(p):]
2457                     break
2458
2459         path = wildcard_decode(path)
2460         return path
2461
2462     def splitFilesIntoBranches(self, commit):
2463         """Look at each depotFile in the commit to figure out to what
2464            branch it belongs."""
2465
2466         if self.clientSpecDirs:
2467             files = self.extractFilesFromCommit(commit)
2468             self.clientSpecDirs.update_client_spec_path_cache(files)
2469
2470         branches = {}
2471         fnum = 0
2472         while commit.has_key("depotFile%s" % fnum):
2473             path =  commit["depotFile%s" % fnum]
2474             found = [p for p in self.depotPaths
2475                      if p4PathStartsWith(path, p)]
2476             if not found:
2477                 fnum = fnum + 1
2478                 continue
2479
2480             file = {}
2481             file["path"] = path
2482             file["rev"] = commit["rev%s" % fnum]
2483             file["action"] = commit["action%s" % fnum]
2484             file["type"] = commit["type%s" % fnum]
2485             fnum = fnum + 1
2486
2487             # start with the full relative path where this file would
2488             # go in a p4 client
2489             if self.useClientSpec:
2490                 relPath = self.clientSpecDirs.map_in_client(path)
2491             else:
2492                 relPath = self.stripRepoPath(path, self.depotPaths)
2493
2494             for branch in self.knownBranches.keys():
2495                 # add a trailing slash so that a commit into qt/4.2foo
2496                 # doesn't end up in qt/4.2, e.g.
2497                 if relPath.startswith(branch + "/"):
2498                     if branch not in branches:
2499                         branches[branch] = []
2500                     branches[branch].append(file)
2501                     break
2502
2503         return branches
2504
2505     def writeToGitStream(self, gitMode, relPath, contents):
2506         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2507         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2508         for d in contents:
2509             self.gitStream.write(d)
2510         self.gitStream.write('\n')
2511
2512     def encodeWithUTF8(self, path):
2513         try:
2514             path.decode('ascii')
2515         except:
2516             encoding = 'utf8'
2517             if gitConfig('git-p4.pathEncoding'):
2518                 encoding = gitConfig('git-p4.pathEncoding')
2519             path = path.decode(encoding, 'replace').encode('utf8', 'replace')
2520             if self.verbose:
2521                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, path)
2522         return path
2523
2524     # output one file from the P4 stream
2525     # - helper for streamP4Files
2526
2527     def streamOneP4File(self, file, contents):
2528         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2529         relPath = self.encodeWithUTF8(relPath)
2530         if verbose:
2531             size = int(self.stream_file['fileSize'])
2532             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2533             sys.stdout.flush()
2534
2535         (type_base, type_mods) = split_p4_type(file["type"])
2536
2537         git_mode = "100644"
2538         if "x" in type_mods:
2539             git_mode = "100755"
2540         if type_base == "symlink":
2541             git_mode = "120000"
2542             # p4 print on a symlink sometimes contains "target\n";
2543             # if it does, remove the newline
2544             data = ''.join(contents)
2545             if not data:
2546                 # Some version of p4 allowed creating a symlink that pointed
2547                 # to nothing.  This causes p4 errors when checking out such
2548                 # a change, and errors here too.  Work around it by ignoring
2549                 # the bad symlink; hopefully a future change fixes it.
2550                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2551                 return
2552             elif data[-1] == '\n':
2553                 contents = [data[:-1]]
2554             else:
2555                 contents = [data]
2556
2557         if type_base == "utf16":
2558             # p4 delivers different text in the python output to -G
2559             # than it does when using "print -o", or normal p4 client
2560             # operations.  utf16 is converted to ascii or utf8, perhaps.
2561             # But ascii text saved as -t utf16 is completely mangled.
2562             # Invoke print -o to get the real contents.
2563             #
2564             # On windows, the newlines will always be mangled by print, so put
2565             # them back too.  This is not needed to the cygwin windows version,
2566             # just the native "NT" type.
2567             #
2568             try:
2569                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2570             except Exception as e:
2571                 if 'Translation of file content failed' in str(e):
2572                     type_base = 'binary'
2573                 else:
2574                     raise e
2575             else:
2576                 if p4_version_string().find('/NT') >= 0:
2577                     text = text.replace('\r\n', '\n')
2578                 contents = [ text ]
2579
2580         if type_base == "apple":
2581             # Apple filetype files will be streamed as a concatenation of
2582             # its appledouble header and the contents.  This is useless
2583             # on both macs and non-macs.  If using "print -q -o xx", it
2584             # will create "xx" with the data, and "%xx" with the header.
2585             # This is also not very useful.
2586             #
2587             # Ideally, someday, this script can learn how to generate
2588             # appledouble files directly and import those to git, but
2589             # non-mac machines can never find a use for apple filetype.
2590             print "\nIgnoring apple filetype file %s" % file['depotFile']
2591             return
2592
2593         # Note that we do not try to de-mangle keywords on utf16 files,
2594         # even though in theory somebody may want that.
2595         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2596         if pattern:
2597             regexp = re.compile(pattern, re.VERBOSE)
2598             text = ''.join(contents)
2599             text = regexp.sub(r'$\1$', text)
2600             contents = [ text ]
2601
2602         if self.largeFileSystem:
2603             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2604
2605         self.writeToGitStream(git_mode, relPath, contents)
2606
2607     def streamOneP4Deletion(self, file):
2608         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2609         relPath = self.encodeWithUTF8(relPath)
2610         if verbose:
2611             sys.stdout.write("delete %s\n" % relPath)
2612             sys.stdout.flush()
2613         self.gitStream.write("D %s\n" % relPath)
2614
2615         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2616             self.largeFileSystem.removeLargeFile(relPath)
2617
2618     # handle another chunk of streaming data
2619     def streamP4FilesCb(self, marshalled):
2620
2621         # catch p4 errors and complain
2622         err = None
2623         if "code" in marshalled:
2624             if marshalled["code"] == "error":
2625                 if "data" in marshalled:
2626                     err = marshalled["data"].rstrip()
2627
2628         if not err and 'fileSize' in self.stream_file:
2629             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2630             if required_bytes > 0:
2631                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2632                     os.getcwd(), required_bytes/1024/1024
2633                 )
2634
2635         if err:
2636             f = None
2637             if self.stream_have_file_info:
2638                 if "depotFile" in self.stream_file:
2639                     f = self.stream_file["depotFile"]
2640             # force a failure in fast-import, else an empty
2641             # commit will be made
2642             self.gitStream.write("\n")
2643             self.gitStream.write("die-now\n")
2644             self.gitStream.close()
2645             # ignore errors, but make sure it exits first
2646             self.importProcess.wait()
2647             if f:
2648                 die("Error from p4 print for %s: %s" % (f, err))
2649             else:
2650                 die("Error from p4 print: %s" % err)
2651
2652         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2653             # start of a new file - output the old one first
2654             self.streamOneP4File(self.stream_file, self.stream_contents)
2655             self.stream_file = {}
2656             self.stream_contents = []
2657             self.stream_have_file_info = False
2658
2659         # pick up the new file information... for the
2660         # 'data' field we need to append to our array
2661         for k in marshalled.keys():
2662             if k == 'data':
2663                 if 'streamContentSize' not in self.stream_file:
2664                     self.stream_file['streamContentSize'] = 0
2665                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2666                 self.stream_contents.append(marshalled['data'])
2667             else:
2668                 self.stream_file[k] = marshalled[k]
2669
2670         if (verbose and
2671             'streamContentSize' in self.stream_file and
2672             'fileSize' in self.stream_file and
2673             'depotFile' in self.stream_file):
2674             size = int(self.stream_file["fileSize"])
2675             if size > 0:
2676                 progress = 100*self.stream_file['streamContentSize']/size
2677                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2678                 sys.stdout.flush()
2679
2680         self.stream_have_file_info = True
2681
2682     # Stream directly from "p4 files" into "git fast-import"
2683     def streamP4Files(self, files):
2684         filesForCommit = []
2685         filesToRead = []
2686         filesToDelete = []
2687
2688         for f in files:
2689             filesForCommit.append(f)
2690             if f['action'] in self.delete_actions:
2691                 filesToDelete.append(f)
2692             else:
2693                 filesToRead.append(f)
2694
2695         # deleted files...
2696         for f in filesToDelete:
2697             self.streamOneP4Deletion(f)
2698
2699         if len(filesToRead) > 0:
2700             self.stream_file = {}
2701             self.stream_contents = []
2702             self.stream_have_file_info = False
2703
2704             # curry self argument
2705             def streamP4FilesCbSelf(entry):
2706                 self.streamP4FilesCb(entry)
2707
2708             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2709
2710             p4CmdList(["-x", "-", "print"],
2711                       stdin=fileArgs,
2712                       cb=streamP4FilesCbSelf)
2713
2714             # do the last chunk
2715             if self.stream_file.has_key('depotFile'):
2716                 self.streamOneP4File(self.stream_file, self.stream_contents)
2717
2718     def make_email(self, userid):
2719         if userid in self.users:
2720             return self.users[userid]
2721         else:
2722             return "%s <a@b>" % userid
2723
2724     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2725         """ Stream a p4 tag.
2726         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2727         """
2728
2729         if verbose:
2730             print "writing tag %s for commit %s" % (labelName, commit)
2731         gitStream.write("tag %s\n" % labelName)
2732         gitStream.write("from %s\n" % commit)
2733
2734         if labelDetails.has_key('Owner'):
2735             owner = labelDetails["Owner"]
2736         else:
2737             owner = None
2738
2739         # Try to use the owner of the p4 label, or failing that,
2740         # the current p4 user id.
2741         if owner:
2742             email = self.make_email(owner)
2743         else:
2744             email = self.make_email(self.p4UserId())
2745         tagger = "%s %s %s" % (email, epoch, self.tz)
2746
2747         gitStream.write("tagger %s\n" % tagger)
2748
2749         print "labelDetails=",labelDetails
2750         if labelDetails.has_key('Description'):
2751             description = labelDetails['Description']
2752         else:
2753             description = 'Label from git p4'
2754
2755         gitStream.write("data %d\n" % len(description))
2756         gitStream.write(description)
2757         gitStream.write("\n")
2758
2759     def inClientSpec(self, path):
2760         if not self.clientSpecDirs:
2761             return True
2762         inClientSpec = self.clientSpecDirs.map_in_client(path)
2763         if not inClientSpec and self.verbose:
2764             print('Ignoring file outside of client spec: {0}'.format(path))
2765         return inClientSpec
2766
2767     def hasBranchPrefix(self, path):
2768         if not self.branchPrefixes:
2769             return True
2770         hasPrefix = [p for p in self.branchPrefixes
2771                         if p4PathStartsWith(path, p)]
2772         if not hasPrefix and self.verbose:
2773             print('Ignoring file outside of prefix: {0}'.format(path))
2774         return hasPrefix
2775
2776     def commit(self, details, files, branch, parent = ""):
2777         epoch = details["time"]
2778         author = details["user"]
2779         jobs = self.extractJobsFromCommit(details)
2780
2781         if self.verbose:
2782             print('commit into {0}'.format(branch))
2783
2784         if self.clientSpecDirs:
2785             self.clientSpecDirs.update_client_spec_path_cache(files)
2786
2787         files = [f for f in files
2788             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2789
2790         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2791             print('Ignoring revision {0} as it would produce an empty commit.'
2792                 .format(details['change']))
2793             return
2794
2795         self.gitStream.write("commit %s\n" % branch)
2796         self.gitStream.write("mark :%s\n" % details["change"])
2797         self.committedChanges.add(int(details["change"]))
2798         committer = ""
2799         if author not in self.users:
2800             self.getUserMapFromPerforceServer()
2801         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2802
2803         self.gitStream.write("committer %s\n" % committer)
2804
2805         self.gitStream.write("data <<EOT\n")
2806         self.gitStream.write(details["desc"])
2807         if len(jobs) > 0:
2808             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2809         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2810                              (','.join(self.branchPrefixes), details["change"]))
2811         if len(details['options']) > 0:
2812             self.gitStream.write(": options = %s" % details['options'])
2813         self.gitStream.write("]\nEOT\n\n")
2814
2815         if len(parent) > 0:
2816             if self.verbose:
2817                 print "parent %s" % parent
2818             self.gitStream.write("from %s\n" % parent)
2819
2820         self.streamP4Files(files)
2821         self.gitStream.write("\n")
2822
2823         change = int(details["change"])
2824
2825         if self.labels.has_key(change):
2826             label = self.labels[change]
2827             labelDetails = label[0]
2828             labelRevisions = label[1]
2829             if self.verbose:
2830                 print "Change %s is labelled %s" % (change, labelDetails)
2831
2832             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2833                                                 for p in self.branchPrefixes])
2834
2835             if len(files) == len(labelRevisions):
2836
2837                 cleanedFiles = {}
2838                 for info in files:
2839                     if info["action"] in self.delete_actions:
2840                         continue
2841                     cleanedFiles[info["depotFile"]] = info["rev"]
2842
2843                 if cleanedFiles == labelRevisions:
2844                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2845
2846                 else:
2847                     if not self.silent:
2848                         print ("Tag %s does not match with change %s: files do not match."
2849                                % (labelDetails["label"], change))
2850
2851             else:
2852                 if not self.silent:
2853                     print ("Tag %s does not match with change %s: file count is different."
2854                            % (labelDetails["label"], change))
2855
2856     # Build a dictionary of changelists and labels, for "detect-labels" option.
2857     def getLabels(self):
2858         self.labels = {}
2859
2860         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2861         if len(l) > 0 and not self.silent:
2862             print "Finding files belonging to labels in %s" % `self.depotPaths`
2863
2864         for output in l:
2865             label = output["label"]
2866             revisions = {}
2867             newestChange = 0
2868             if self.verbose:
2869                 print "Querying files for label %s" % label
2870             for file in p4CmdList(["files"] +
2871                                       ["%s...@%s" % (p, label)
2872                                           for p in self.depotPaths]):
2873                 revisions[file["depotFile"]] = file["rev"]
2874                 change = int(file["change"])
2875                 if change > newestChange:
2876                     newestChange = change
2877
2878             self.labels[newestChange] = [output, revisions]
2879
2880         if self.verbose:
2881             print "Label changes: %s" % self.labels.keys()
2882
2883     # Import p4 labels as git tags. A direct mapping does not
2884     # exist, so assume that if all the files are at the same revision
2885     # then we can use that, or it's something more complicated we should
2886     # just ignore.
2887     def importP4Labels(self, stream, p4Labels):
2888         if verbose:
2889             print "import p4 labels: " + ' '.join(p4Labels)
2890
2891         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2892         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2893         if len(validLabelRegexp) == 0:
2894             validLabelRegexp = defaultLabelRegexp
2895         m = re.compile(validLabelRegexp)
2896
2897         for name in p4Labels:
2898             commitFound = False
2899
2900             if not m.match(name):
2901                 if verbose:
2902                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2903                 continue
2904
2905             if name in ignoredP4Labels:
2906                 continue
2907
2908             labelDetails = p4CmdList(['label', "-o", name])[0]
2909
2910             # get the most recent changelist for each file in this label
2911             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2912                                 for p in self.depotPaths])
2913
2914             if change.has_key('change'):
2915                 # find the corresponding git commit; take the oldest commit
2916                 changelist = int(change['change'])
2917                 if changelist in self.committedChanges:
2918                     gitCommit = ":%d" % changelist       # use a fast-import mark
2919                     commitFound = True
2920                 else:
2921                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2922                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2923                     if len(gitCommit) == 0:
2924                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2925                     else:
2926                         commitFound = True
2927                         gitCommit = gitCommit.strip()
2928
2929                 if commitFound:
2930                     # Convert from p4 time format
2931                     try:
2932                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2933                     except ValueError:
2934                         print "Could not convert label time %s" % labelDetails['Update']
2935                         tmwhen = 1
2936
2937                     when = int(time.mktime(tmwhen))
2938                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2939                     if verbose:
2940                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2941             else:
2942                 if verbose:
2943                     print "Label %s has no changelists - possibly deleted?" % name
2944
2945             if not commitFound:
2946                 # We can't import this label; don't try again as it will get very
2947                 # expensive repeatedly fetching all the files for labels that will
2948                 # never be imported. If the label is moved in the future, the
2949                 # ignore will need to be removed manually.
2950                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2951
2952     def guessProjectName(self):
2953         for p in self.depotPaths:
2954             if p.endswith("/"):
2955                 p = p[:-1]
2956             p = p[p.strip().rfind("/") + 1:]
2957             if not p.endswith("/"):
2958                p += "/"
2959             return p
2960
2961     def getBranchMapping(self):
2962         lostAndFoundBranches = set()
2963
2964         user = gitConfig("git-p4.branchUser")
2965         if len(user) > 0:
2966             command = "branches -u %s" % user
2967         else:
2968             command = "branches"
2969
2970         for info in p4CmdList(command):
2971             details = p4Cmd(["branch", "-o", info["branch"]])
2972             viewIdx = 0
2973             while details.has_key("View%s" % viewIdx):
2974                 paths = details["View%s" % viewIdx].split(" ")
2975                 viewIdx = viewIdx + 1
2976                 # require standard //depot/foo/... //depot/bar/... mapping
2977                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2978                     continue
2979                 source = paths[0]
2980                 destination = paths[1]
2981                 ## HACK
2982                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2983                     source = source[len(self.depotPaths[0]):-4]
2984                     destination = destination[len(self.depotPaths[0]):-4]
2985
2986                     if destination in self.knownBranches:
2987                         if not self.silent:
2988                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2989                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2990                         continue
2991
2992                     self.knownBranches[destination] = source
2993
2994                     lostAndFoundBranches.discard(destination)
2995
2996                     if source not in self.knownBranches:
2997                         lostAndFoundBranches.add(source)
2998
2999         # Perforce does not strictly require branches to be defined, so we also
3000         # check git config for a branch list.
3001         #
3002         # Example of branch definition in git config file:
3003         # [git-p4]
3004         #   branchList=main:branchA
3005         #   branchList=main:branchB
3006         #   branchList=branchA:branchC
3007         configBranches = gitConfigList("git-p4.branchList")
3008         for branch in configBranches:
3009             if branch:
3010                 (source, destination) = branch.split(":")
3011                 self.knownBranches[destination] = source
3012
3013                 lostAndFoundBranches.discard(destination)
3014
3015                 if source not in self.knownBranches:
3016                     lostAndFoundBranches.add(source)
3017
3018
3019         for branch in lostAndFoundBranches:
3020             self.knownBranches[branch] = branch
3021
3022     def getBranchMappingFromGitBranches(self):
3023         branches = p4BranchesInGit(self.importIntoRemotes)
3024         for branch in branches.keys():
3025             if branch == "master":
3026                 branch = "main"
3027             else:
3028                 branch = branch[len(self.projectName):]
3029             self.knownBranches[branch] = branch
3030
3031     def updateOptionDict(self, d):
3032         option_keys = {}
3033         if self.keepRepoPath:
3034             option_keys['keepRepoPath'] = 1
3035
3036         d["options"] = ' '.join(sorted(option_keys.keys()))
3037
3038     def readOptions(self, d):
3039         self.keepRepoPath = (d.has_key('options')
3040                              and ('keepRepoPath' in d['options']))
3041
3042     def gitRefForBranch(self, branch):
3043         if branch == "main":
3044             return self.refPrefix + "master"
3045
3046         if len(branch) <= 0:
3047             return branch
3048
3049         return self.refPrefix + self.projectName + branch
3050
3051     def gitCommitByP4Change(self, ref, change):
3052         if self.verbose:
3053             print "looking in ref " + ref + " for change %s using bisect..." % change
3054
3055         earliestCommit = ""
3056         latestCommit = parseRevision(ref)
3057
3058         while True:
3059             if self.verbose:
3060                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
3061             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
3062             if len(next) == 0:
3063                 if self.verbose:
3064                     print "argh"
3065                 return ""
3066             log = extractLogMessageFromGitCommit(next)
3067             settings = extractSettingsGitLog(log)
3068             currentChange = int(settings['change'])
3069             if self.verbose:
3070                 print "current change %s" % currentChange
3071
3072             if currentChange == change:
3073                 if self.verbose:
3074                     print "found %s" % next
3075                 return next
3076
3077             if currentChange < change:
3078                 earliestCommit = "^%s" % next
3079             else:
3080                 latestCommit = "%s" % next
3081
3082         return ""
3083
3084     def importNewBranch(self, branch, maxChange):
3085         # make fast-import flush all changes to disk and update the refs using the checkpoint
3086         # command so that we can try to find the branch parent in the git history
3087         self.gitStream.write("checkpoint\n\n");
3088         self.gitStream.flush();
3089         branchPrefix = self.depotPaths[0] + branch + "/"
3090         range = "@1,%s" % maxChange
3091         #print "prefix" + branchPrefix
3092         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3093         if len(changes) <= 0:
3094             return False
3095         firstChange = changes[0]
3096         #print "first change in branch: %s" % firstChange
3097         sourceBranch = self.knownBranches[branch]
3098         sourceDepotPath = self.depotPaths[0] + sourceBranch
3099         sourceRef = self.gitRefForBranch(sourceBranch)
3100         #print "source " + sourceBranch
3101
3102         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3103         #print "branch parent: %s" % branchParentChange
3104         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3105         if len(gitParent) > 0:
3106             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3107             #print "parent git commit: %s" % gitParent
3108
3109         self.importChanges(changes)
3110         return True
3111
3112     def searchParent(self, parent, branch, target):
3113         parentFound = False
3114         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3115                                      "--no-merges", parent]):
3116             blob = blob.strip()
3117             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3118                 parentFound = True
3119                 if self.verbose:
3120                     print "Found parent of %s in commit %s" % (branch, blob)
3121                 break
3122         if parentFound:
3123             return blob
3124         else:
3125             return None
3126
3127     def importChanges(self, changes):
3128         cnt = 1
3129         for change in changes:
3130             description = p4_describe(change)
3131             self.updateOptionDict(description)
3132
3133             if not self.silent:
3134                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3135                 sys.stdout.flush()
3136             cnt = cnt + 1
3137
3138             try:
3139                 if self.detectBranches:
3140                     branches = self.splitFilesIntoBranches(description)
3141                     for branch in branches.keys():
3142                         ## HACK  --hwn
3143                         branchPrefix = self.depotPaths[0] + branch + "/"
3144                         self.branchPrefixes = [ branchPrefix ]
3145
3146                         parent = ""
3147
3148                         filesForCommit = branches[branch]
3149
3150                         if self.verbose:
3151                             print "branch is %s" % branch
3152
3153                         self.updatedBranches.add(branch)
3154
3155                         if branch not in self.createdBranches:
3156                             self.createdBranches.add(branch)
3157                             parent = self.knownBranches[branch]
3158                             if parent == branch:
3159                                 parent = ""
3160                             else:
3161                                 fullBranch = self.projectName + branch
3162                                 if fullBranch not in self.p4BranchesInGit:
3163                                     if not self.silent:
3164                                         print("\n    Importing new branch %s" % fullBranch);
3165                                     if self.importNewBranch(branch, change - 1):
3166                                         parent = ""
3167                                         self.p4BranchesInGit.append(fullBranch)
3168                                     if not self.silent:
3169                                         print("\n    Resuming with change %s" % change);
3170
3171                                 if self.verbose:
3172                                     print "parent determined through known branches: %s" % parent
3173
3174                         branch = self.gitRefForBranch(branch)
3175                         parent = self.gitRefForBranch(parent)
3176
3177                         if self.verbose:
3178                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3179
3180                         if len(parent) == 0 and branch in self.initialParents:
3181                             parent = self.initialParents[branch]
3182                             del self.initialParents[branch]
3183
3184                         blob = None
3185                         if len(parent) > 0:
3186                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3187                             if self.verbose:
3188                                 print "Creating temporary branch: " + tempBranch
3189                             self.commit(description, filesForCommit, tempBranch)
3190                             self.tempBranches.append(tempBranch)
3191                             self.checkpoint()
3192                             blob = self.searchParent(parent, branch, tempBranch)
3193                         if blob:
3194                             self.commit(description, filesForCommit, branch, blob)
3195                         else:
3196                             if self.verbose:
3197                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3198                             self.commit(description, filesForCommit, branch, parent)
3199                 else:
3200                     files = self.extractFilesFromCommit(description)
3201                     self.commit(description, files, self.branch,
3202                                 self.initialParent)
3203                     # only needed once, to connect to the previous commit
3204                     self.initialParent = ""
3205             except IOError:
3206                 print self.gitError.read()
3207                 sys.exit(1)
3208
3209     def importHeadRevision(self, revision):
3210         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3211
3212         details = {}
3213         details["user"] = "git perforce import user"
3214         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3215                            % (' '.join(self.depotPaths), revision))
3216         details["change"] = revision
3217         newestRevision = 0
3218
3219         fileCnt = 0
3220         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3221
3222         for info in p4CmdList(["files"] + fileArgs):
3223
3224             if 'code' in info and info['code'] == 'error':
3225                 sys.stderr.write("p4 returned an error: %s\n"
3226                                  % info['data'])
3227                 if info['data'].find("must refer to client") >= 0:
3228                     sys.stderr.write("This particular p4 error is misleading.\n")
3229                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3230                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3231                 sys.exit(1)
3232             if 'p4ExitCode' in info:
3233                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3234                 sys.exit(1)
3235
3236
3237             change = int(info["change"])
3238             if change > newestRevision:
3239                 newestRevision = change
3240
3241             if info["action"] in self.delete_actions:
3242                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3243                 #fileCnt = fileCnt + 1
3244                 continue
3245
3246             for prop in ["depotFile", "rev", "action", "type" ]:
3247                 details["%s%s" % (prop, fileCnt)] = info[prop]
3248
3249             fileCnt = fileCnt + 1
3250
3251         details["change"] = newestRevision
3252
3253         # Use time from top-most change so that all git p4 clones of
3254         # the same p4 repo have the same commit SHA1s.
3255         res = p4_describe(newestRevision)
3256         details["time"] = res["time"]
3257
3258         self.updateOptionDict(details)
3259         try:
3260             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3261         except IOError:
3262             print "IO error with git fast-import. Is your git version recent enough?"
3263             print self.gitError.read()
3264
3265
3266     def run(self, args):
3267         self.depotPaths = []
3268         self.changeRange = ""
3269         self.previousDepotPaths = []
3270         self.hasOrigin = False
3271
3272         # map from branch depot path to parent branch
3273         self.knownBranches = {}
3274         self.initialParents = {}
3275
3276         if self.importIntoRemotes:
3277             self.refPrefix = "refs/remotes/p4/"
3278         else:
3279             self.refPrefix = "refs/heads/p4/"
3280
3281         if self.syncWithOrigin:
3282             self.hasOrigin = originP4BranchesExist()
3283             if self.hasOrigin:
3284                 if not self.silent:
3285                     print 'Syncing with origin first, using "git fetch origin"'
3286                 system("git fetch origin")
3287
3288         branch_arg_given = bool(self.branch)
3289         if len(self.branch) == 0:
3290             self.branch = self.refPrefix + "master"
3291             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3292                 system("git update-ref %s refs/heads/p4" % self.branch)
3293                 system("git branch -D p4")
3294
3295         # accept either the command-line option, or the configuration variable
3296         if self.useClientSpec:
3297             # will use this after clone to set the variable
3298             self.useClientSpec_from_options = True
3299         else:
3300             if gitConfigBool("git-p4.useclientspec"):
3301                 self.useClientSpec = True
3302         if self.useClientSpec:
3303             self.clientSpecDirs = getClientSpec()
3304
3305         # TODO: should always look at previous commits,
3306         # merge with previous imports, if possible.
3307         if args == []:
3308             if self.hasOrigin:
3309                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3310
3311             # branches holds mapping from branch name to sha1
3312             branches = p4BranchesInGit(self.importIntoRemotes)
3313
3314             # restrict to just this one, disabling detect-branches
3315             if branch_arg_given:
3316                 short = self.branch.split("/")[-1]
3317                 if short in branches:
3318                     self.p4BranchesInGit = [ short ]
3319             else:
3320                 self.p4BranchesInGit = branches.keys()
3321
3322             if len(self.p4BranchesInGit) > 1:
3323                 if not self.silent:
3324                     print "Importing from/into multiple branches"
3325                 self.detectBranches = True
3326                 for branch in branches.keys():
3327                     self.initialParents[self.refPrefix + branch] = \
3328                         branches[branch]
3329
3330             if self.verbose:
3331                 print "branches: %s" % self.p4BranchesInGit
3332
3333             p4Change = 0
3334             for branch in self.p4BranchesInGit:
3335                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3336
3337                 settings = extractSettingsGitLog(logMsg)
3338
3339                 self.readOptions(settings)
3340                 if (settings.has_key('depot-paths')
3341                     and settings.has_key ('change')):
3342                     change = int(settings['change']) + 1
3343                     p4Change = max(p4Change, change)
3344
3345                     depotPaths = sorted(settings['depot-paths'])
3346                     if self.previousDepotPaths == []:
3347                         self.previousDepotPaths = depotPaths
3348                     else:
3349                         paths = []
3350                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3351                             prev_list = prev.split("/")
3352                             cur_list = cur.split("/")
3353                             for i in range(0, min(len(cur_list), len(prev_list))):
3354                                 if cur_list[i] <> prev_list[i]:
3355                                     i = i - 1
3356                                     break
3357
3358                             paths.append ("/".join(cur_list[:i + 1]))
3359
3360                         self.previousDepotPaths = paths
3361
3362             if p4Change > 0:
3363                 self.depotPaths = sorted(self.previousDepotPaths)
3364                 self.changeRange = "@%s,#head" % p4Change
3365                 if not self.silent and not self.detectBranches:
3366                     print "Performing incremental import into %s git branch" % self.branch
3367
3368         # accept multiple ref name abbreviations:
3369         #    refs/foo/bar/branch -> use it exactly
3370         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3371         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3372         if not self.branch.startswith("refs/"):
3373             if self.importIntoRemotes:
3374                 prepend = "refs/remotes/"
3375             else:
3376                 prepend = "refs/heads/"
3377             if not self.branch.startswith("p4/"):
3378                 prepend += "p4/"
3379             self.branch = prepend + self.branch
3380
3381         if len(args) == 0 and self.depotPaths:
3382             if not self.silent:
3383                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3384         else:
3385             if self.depotPaths and self.depotPaths != args:
3386                 print ("previous import used depot path %s and now %s was specified. "
3387                        "This doesn't work!" % (' '.join (self.depotPaths),
3388                                                ' '.join (args)))
3389                 sys.exit(1)
3390
3391             self.depotPaths = sorted(args)
3392
3393         revision = ""
3394         self.users = {}
3395
3396         # Make sure no revision specifiers are used when --changesfile
3397         # is specified.
3398         bad_changesfile = False
3399         if len(self.changesFile) > 0:
3400             for p in self.depotPaths:
3401                 if p.find("@") >= 0 or p.find("#") >= 0:
3402                     bad_changesfile = True
3403                     break
3404         if bad_changesfile:
3405             die("Option --changesfile is incompatible with revision specifiers")
3406
3407         newPaths = []
3408         for p in self.depotPaths:
3409             if p.find("@") != -1:
3410                 atIdx = p.index("@")
3411                 self.changeRange = p[atIdx:]
3412                 if self.changeRange == "@all":
3413                     self.changeRange = ""
3414                 elif ',' not in self.changeRange:
3415                     revision = self.changeRange
3416                     self.changeRange = ""
3417                 p = p[:atIdx]
3418             elif p.find("#") != -1:
3419                 hashIdx = p.index("#")
3420                 revision = p[hashIdx:]
3421                 p = p[:hashIdx]
3422             elif self.previousDepotPaths == []:
3423                 # pay attention to changesfile, if given, else import
3424                 # the entire p4 tree at the head revision
3425                 if len(self.changesFile) == 0:
3426                     revision = "#head"
3427
3428             p = re.sub ("\.\.\.$", "", p)
3429             if not p.endswith("/"):
3430                 p += "/"
3431
3432             newPaths.append(p)
3433
3434         self.depotPaths = newPaths
3435
3436         # --detect-branches may change this for each branch
3437         self.branchPrefixes = self.depotPaths
3438
3439         self.loadUserMapFromCache()
3440         self.labels = {}
3441         if self.detectLabels:
3442             self.getLabels();
3443
3444         if self.detectBranches:
3445             ## FIXME - what's a P4 projectName ?
3446             self.projectName = self.guessProjectName()
3447
3448             if self.hasOrigin:
3449                 self.getBranchMappingFromGitBranches()
3450             else:
3451                 self.getBranchMapping()
3452             if self.verbose:
3453                 print "p4-git branches: %s" % self.p4BranchesInGit
3454                 print "initial parents: %s" % self.initialParents
3455             for b in self.p4BranchesInGit:
3456                 if b != "master":
3457
3458                     ## FIXME
3459                     b = b[len(self.projectName):]
3460                 self.createdBranches.add(b)
3461
3462         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3463
3464         self.importProcess = subprocess.Popen(["git", "fast-import"],
3465                                               stdin=subprocess.PIPE,
3466                                               stdout=subprocess.PIPE,
3467                                               stderr=subprocess.PIPE);
3468         self.gitOutput = self.importProcess.stdout
3469         self.gitStream = self.importProcess.stdin
3470         self.gitError = self.importProcess.stderr
3471
3472         if revision:
3473             self.importHeadRevision(revision)
3474         else:
3475             changes = []
3476
3477             if len(self.changesFile) > 0:
3478                 output = open(self.changesFile).readlines()
3479                 changeSet = set()
3480                 for line in output:
3481                     changeSet.add(int(line))
3482
3483                 for change in changeSet:
3484                     changes.append(change)
3485
3486                 changes.sort()
3487             else:
3488                 # catch "git p4 sync" with no new branches, in a repo that
3489                 # does not have any existing p4 branches
3490                 if len(args) == 0:
3491                     if not self.p4BranchesInGit:
3492                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3493
3494                     # The default branch is master, unless --branch is used to
3495                     # specify something else.  Make sure it exists, or complain
3496                     # nicely about how to use --branch.
3497                     if not self.detectBranches:
3498                         if not branch_exists(self.branch):
3499                             if branch_arg_given:
3500                                 die("Error: branch %s does not exist." % self.branch)
3501                             else:
3502                                 die("Error: no branch %s; perhaps specify one with --branch." %
3503                                     self.branch)
3504
3505                 if self.verbose:
3506                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3507                                                               self.changeRange)
3508                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3509
3510                 if len(self.maxChanges) > 0:
3511                     changes = changes[:min(int(self.maxChanges), len(changes))]
3512
3513             if len(changes) == 0:
3514                 if not self.silent:
3515                     print "No changes to import!"
3516             else:
3517                 if not self.silent and not self.detectBranches:
3518                     print "Import destination: %s" % self.branch
3519
3520                 self.updatedBranches = set()
3521
3522                 if not self.detectBranches:
3523                     if args:
3524                         # start a new branch
3525                         self.initialParent = ""
3526                     else:
3527                         # build on a previous revision
3528                         self.initialParent = parseRevision(self.branch)
3529
3530                 self.importChanges(changes)
3531
3532                 if not self.silent:
3533                     print ""
3534                     if len(self.updatedBranches) > 0:
3535                         sys.stdout.write("Updated branches: ")
3536                         for b in self.updatedBranches:
3537                             sys.stdout.write("%s " % b)
3538                         sys.stdout.write("\n")
3539
3540         if gitConfigBool("git-p4.importLabels"):
3541             self.importLabels = True
3542
3543         if self.importLabels:
3544             p4Labels = getP4Labels(self.depotPaths)
3545             gitTags = getGitTags()
3546
3547             missingP4Labels = p4Labels - gitTags
3548             self.importP4Labels(self.gitStream, missingP4Labels)
3549
3550         self.gitStream.close()
3551         if self.importProcess.wait() != 0:
3552             die("fast-import failed: %s" % self.gitError.read())
3553         self.gitOutput.close()
3554         self.gitError.close()
3555
3556         # Cleanup temporary branches created during import
3557         if self.tempBranches != []:
3558             for branch in self.tempBranches:
3559                 read_pipe("git update-ref -d %s" % branch)
3560             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3561
3562         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3563         # a convenient shortcut refname "p4".
3564         if self.importIntoRemotes:
3565             head_ref = self.refPrefix + "HEAD"
3566             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3567                 system(["git", "symbolic-ref", head_ref, self.branch])
3568
3569         return True
3570
3571 class P4Rebase(Command):
3572     def __init__(self):
3573         Command.__init__(self)
3574         self.options = [
3575                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3576         ]
3577         self.importLabels = False
3578         self.description = ("Fetches the latest revision from perforce and "
3579                             + "rebases the current work (branch) against it")
3580
3581     def run(self, args):
3582         sync = P4Sync()
3583         sync.importLabels = self.importLabels
3584         sync.run([])
3585
3586         return self.rebase()
3587
3588     def rebase(self):
3589         if os.system("git update-index --refresh") != 0:
3590             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3591         if len(read_pipe("git diff-index HEAD --")) > 0:
3592             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3593
3594         [upstream, settings] = findUpstreamBranchPoint()
3595         if len(upstream) == 0:
3596             die("Cannot find upstream branchpoint for rebase")
3597
3598         # the branchpoint may be p4/foo~3, so strip off the parent
3599         upstream = re.sub("~[0-9]+$", "", upstream)
3600
3601         print "Rebasing the current branch onto %s" % upstream
3602         oldHead = read_pipe("git rev-parse HEAD").strip()
3603         system("git rebase %s" % upstream)
3604         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3605         return True
3606
3607 class P4Clone(P4Sync):
3608     def __init__(self):
3609         P4Sync.__init__(self)
3610         self.description = "Creates a new git repository and imports from Perforce into it"
3611         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3612         self.options += [
3613             optparse.make_option("--destination", dest="cloneDestination",
3614                                  action='store', default=None,
3615                                  help="where to leave result of the clone"),
3616             optparse.make_option("--bare", dest="cloneBare",
3617                                  action="store_true", default=False),
3618         ]
3619         self.cloneDestination = None
3620         self.needsGit = False
3621         self.cloneBare = False
3622
3623     def defaultDestination(self, args):
3624         ## TODO: use common prefix of args?
3625         depotPath = args[0]
3626         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3627         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3628         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3629         depotDir = re.sub(r"/$", "", depotDir)
3630         return os.path.split(depotDir)[1]
3631
3632     def run(self, args):
3633         if len(args) < 1:
3634             return False
3635
3636         if self.keepRepoPath and not self.cloneDestination:
3637             sys.stderr.write("Must specify destination for --keep-path\n")
3638             sys.exit(1)
3639
3640         depotPaths = args
3641
3642         if not self.cloneDestination and len(depotPaths) > 1:
3643             self.cloneDestination = depotPaths[-1]
3644             depotPaths = depotPaths[:-1]
3645
3646         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3647         for p in depotPaths:
3648             if not p.startswith("//"):
3649                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3650                 return False
3651
3652         if not self.cloneDestination:
3653             self.cloneDestination = self.defaultDestination(args)
3654
3655         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3656
3657         if not os.path.exists(self.cloneDestination):
3658             os.makedirs(self.cloneDestination)
3659         chdir(self.cloneDestination)
3660
3661         init_cmd = [ "git", "init" ]
3662         if self.cloneBare:
3663             init_cmd.append("--bare")
3664         retcode = subprocess.call(init_cmd)
3665         if retcode:
3666             raise CalledProcessError(retcode, init_cmd)
3667
3668         if not P4Sync.run(self, depotPaths):
3669             return False
3670
3671         # create a master branch and check out a work tree
3672         if gitBranchExists(self.branch):
3673             system([ "git", "branch", "master", self.branch ])
3674             if not self.cloneBare:
3675                 system([ "git", "checkout", "-f" ])
3676         else:
3677             print 'Not checking out any branch, use ' \
3678                   '"git checkout -q -b master <branch>"'
3679
3680         # auto-set this variable if invoked with --use-client-spec
3681         if self.useClientSpec_from_options:
3682             system("git config --bool git-p4.useclientspec true")
3683
3684         return True
3685
3686 class P4Branches(Command):
3687     def __init__(self):
3688         Command.__init__(self)
3689         self.options = [ ]
3690         self.description = ("Shows the git branches that hold imports and their "
3691                             + "corresponding perforce depot paths")
3692         self.verbose = False
3693
3694     def run(self, args):
3695         if originP4BranchesExist():
3696             createOrUpdateBranchesFromOrigin()
3697
3698         cmdline = "git rev-parse --symbolic "
3699         cmdline += " --remotes"
3700
3701         for line in read_pipe_lines(cmdline):
3702             line = line.strip()
3703
3704             if not line.startswith('p4/') or line == "p4/HEAD":
3705                 continue
3706             branch = line
3707
3708             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3709             settings = extractSettingsGitLog(log)
3710
3711             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3712         return True
3713
3714 class HelpFormatter(optparse.IndentedHelpFormatter):
3715     def __init__(self):
3716         optparse.IndentedHelpFormatter.__init__(self)
3717
3718     def format_description(self, description):
3719         if description:
3720             return description + "\n"
3721         else:
3722             return ""
3723
3724 def printUsage(commands):
3725     print "usage: %s <command> [options]" % sys.argv[0]
3726     print ""
3727     print "valid commands: %s" % ", ".join(commands)
3728     print ""
3729     print "Try %s <command> --help for command specific help." % sys.argv[0]
3730     print ""
3731
3732 commands = {
3733     "debug" : P4Debug,
3734     "submit" : P4Submit,
3735     "commit" : P4Submit,
3736     "sync" : P4Sync,
3737     "rebase" : P4Rebase,
3738     "clone" : P4Clone,
3739     "rollback" : P4RollBack,
3740     "branches" : P4Branches
3741 }
3742
3743
3744 def main():
3745     if len(sys.argv[1:]) == 0:
3746         printUsage(commands.keys())
3747         sys.exit(2)
3748
3749     cmdName = sys.argv[1]
3750     try:
3751         klass = commands[cmdName]
3752         cmd = klass()
3753     except KeyError:
3754         print "unknown command %s" % cmdName
3755         print ""
3756         printUsage(commands.keys())
3757         sys.exit(2)
3758
3759     options = cmd.options
3760     cmd.gitdir = os.environ.get("GIT_DIR", None)
3761
3762     args = sys.argv[2:]
3763
3764     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3765     if cmd.needsGit:
3766         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3767
3768     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3769                                    options,
3770                                    description = cmd.description,
3771                                    formatter = HelpFormatter())
3772
3773     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3774     global verbose
3775     verbose = cmd.verbose
3776     if cmd.needsGit:
3777         if cmd.gitdir == None:
3778             cmd.gitdir = os.path.abspath(".git")
3779             if not isValidGitDir(cmd.gitdir):
3780                 # "rev-parse --git-dir" without arguments will try $PWD/.git
3781                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3782                 if os.path.exists(cmd.gitdir):
3783                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3784                     if len(cdup) > 0:
3785                         chdir(cdup);
3786
3787         if not isValidGitDir(cmd.gitdir):
3788             if isValidGitDir(cmd.gitdir + "/.git"):
3789                 cmd.gitdir += "/.git"
3790             else:
3791                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3792
3793         # so git commands invoked from the P4 workspace will succeed
3794         os.environ["GIT_DIR"] = cmd.gitdir
3795
3796     if not cmd.run(args):
3797         parser.print_help()
3798         sys.exit(2)
3799
3800
3801 if __name__ == '__main__':
3802     main()