git-p4.py

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10 import sys
  11 if sys.hexversion < 0x02040000:
  12     # The limiter is the subprocess module
  13     sys.stderr.write("git-p4: requires Python 2.4 or later.\n")
  14     sys.exit(1)
  15 import os
  16 import optparse
  17 import marshal
  18 import subprocess
  19 import tempfile
  20 import time
  21 import platform
  22 import re
  23 import shutil
  24 import stat
  25 import zipfile
  26 import zlib
  27 import ctypes
  28 import errno
  29
  30 try:
  31     from subprocess import CalledProcessError
  32 except ImportError:
  33     # from python2.7:subprocess.py
  34     # Exception classes used by this module.
  35     class CalledProcessError(Exception):
  36         """This exception is raised when a process run by check_call() returns
  37         a non-zero exit status.  The exit status will be stored in the
  38         returncode attribute."""
  39         def __init__(self, returncode, cmd):
  40             self.returncode = returncode
  41             self.cmd = cmd
  42         def __str__(self):
  43             return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
  44
  45 verbose = False
  46
  47 # Only labels/tags matching this will be imported/exported
  48 defaultLabelRegexp = r'[a-zA-Z0-9_\-.]+$'
  49
  50 # Grab changes in blocks of this many revisions, unless otherwise requested
  51 defaultBlockSize = 512
  52
  53 def p4_build_cmd(cmd):
  54     """Build a suitable p4 command line.
  55
  56     This consolidates building and returning a p4 command line into one
  57     location. It means that hooking into the environment, or other configuration
  58     can be done more easily.
  59     """
  60     real_cmd = ["p4"]
  61
  62     user = gitConfig("git-p4.user")
  63     if len(user) > 0:
  64         real_cmd += ["-u",user]
  65
  66     password = gitConfig("git-p4.password")
  67     if len(password) > 0:
  68         real_cmd += ["-P", password]
  69
  70     port = gitConfig("git-p4.port")
  71     if len(port) > 0:
  72         real_cmd += ["-p", port]
  73
  74     host = gitConfig("git-p4.host")
  75     if len(host) > 0:
  76         real_cmd += ["-H", host]
  77
  78     client = gitConfig("git-p4.client")
  79     if len(client) > 0:
  80         real_cmd += ["-c", client]
  81
  82
  83     if isinstance(cmd,basestring):
  84         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  85     else:
  86         real_cmd += cmd
  87     return real_cmd
  88
  89 def chdir(path, is_client_path=False):
  90     """Do chdir to the given path, and set the PWD environment
  91        variable for use by P4.  It does not look at getcwd() output.
  92        Since we're not using the shell, it is necessary to set the
  93        PWD environment variable explicitly.
  94
  95        Normally, expand the path to force it to be absolute.  This
  96        addresses the use of relative path names inside P4 settings,
  97        e.g. P4CONFIG=.p4config.  P4 does not simply open the filename
  98        as given; it looks for .p4config using PWD.
  99
 100        If is_client_path, the path was handed to us directly by p4,
 101        and may be a symbolic link.  Do not call os.getcwd() in this
 102        case, because it will cause p4 to think that PWD is not inside
 103        the client path.
 104        """
 105
 106     os.chdir(path)
 107     if not is_client_path:
 108         path = os.getcwd()
 109     os.environ['PWD'] = path
 110
 111 def calcDiskFree():
 112     """Return free space in bytes on the disk of the given dirname."""
 113     if platform.system() == 'Windows':
 114         free_bytes = ctypes.c_ulonglong(0)
 115         ctypes.windll.kernel32.GetDiskFreeSpaceExW(ctypes.c_wchar_p(os.getcwd()), None, None, ctypes.pointer(free_bytes))
 116         return free_bytes.value
 117     else:
 118         st = os.statvfs(os.getcwd())
 119         return st.f_bavail * st.f_frsize
 120
 121 def die(msg):
 122     if verbose:
 123         raise Exception(msg)
 124     else:
 125         sys.stderr.write(msg + "\n")
 126         sys.exit(1)
 127
 128 def write_pipe(c, stdin):
 129     if verbose:
 130         sys.stderr.write('Writing pipe: %s\n' % str(c))
 131
 132     expand = isinstance(c,basestring)
 133     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
 134     pipe = p.stdin
 135     val = pipe.write(stdin)
 136     pipe.close()
 137     if p.wait():
 138         die('Command failed: %s' % str(c))
 139
 140     return val
 141
 142 def p4_write_pipe(c, stdin):
 143     real_cmd = p4_build_cmd(c)
 144     return write_pipe(real_cmd, stdin)
 145
 146 def read_pipe(c, ignore_error=False):
 147     if verbose:
 148         sys.stderr.write('Reading pipe: %s\n' % str(c))
 149
 150     expand = isinstance(c,basestring)
 151     p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=expand)
 152     (out, err) = p.communicate()
 153     if p.returncode != 0 and not ignore_error:
 154         die('Command failed: %s\nError: %s' % (str(c), err))
 155     return out
 156
 157 def p4_read_pipe(c, ignore_error=False):
 158     real_cmd = p4_build_cmd(c)
 159     return read_pipe(real_cmd, ignore_error)
 160
 161 def read_pipe_lines(c):
 162     if verbose:
 163         sys.stderr.write('Reading pipe: %s\n' % str(c))
 164
 165     expand = isinstance(c, basestring)
 166     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 167     pipe = p.stdout
 168     val = pipe.readlines()
 169     if pipe.close() or p.wait():
 170         die('Command failed: %s' % str(c))
 171
 172     return val
 173
 174 def p4_read_pipe_lines(c):
 175     """Specifically invoke p4 on the command supplied. """
 176     real_cmd = p4_build_cmd(c)
 177     return read_pipe_lines(real_cmd)
 178
 179 def p4_has_command(cmd):
 180     """Ask p4 for help on this command.  If it returns an error, the
 181        command does not exist in this version of p4."""
 182     real_cmd = p4_build_cmd(["help", cmd])
 183     p = subprocess.Popen(real_cmd, stdout=subprocess.PIPE,
 184                                    stderr=subprocess.PIPE)
 185     p.communicate()
 186     return p.returncode == 0
 187
 188 def p4_has_move_command():
 189     """See if the move command exists, that it supports -k, and that
 190        it has not been administratively disabled.  The arguments
 191        must be correct, but the filenames do not have to exist.  Use
 192        ones with wildcards so even if they exist, it will fail."""
 193
 194     if not p4_has_command("move"):
 195         return False
 196     cmd = p4_build_cmd(["move", "-k", "@from", "@to"])
 197     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 198     (out, err) = p.communicate()
 199     # return code will be 1 in either case
 200     if err.find("Invalid option") >= 0:
 201         return False
 202     if err.find("disabled") >= 0:
 203         return False
 204     # assume it failed because @... was invalid changelist
 205     return True
 206
 207 def system(cmd, ignore_error=False):
 208     expand = isinstance(cmd,basestring)
 209     if verbose:
 210         sys.stderr.write("executing %s\n" % str(cmd))
 211     retcode = subprocess.call(cmd, shell=expand)
 212     if retcode and not ignore_error:
 213         raise CalledProcessError(retcode, cmd)
 214
 215     return retcode
 216
 217 def p4_system(cmd):
 218     """Specifically invoke p4 as the system command. """
 219     real_cmd = p4_build_cmd(cmd)
 220     expand = isinstance(real_cmd, basestring)
 221     retcode = subprocess.call(real_cmd, shell=expand)
 222     if retcode:
 223         raise CalledProcessError(retcode, real_cmd)
 224
 225 _p4_version_string = None
 226 def p4_version_string():
 227     """Read the version string, showing just the last line, which
 228        hopefully is the interesting version bit.
 229
 230        $ p4 -V
 231        Perforce - The Fast Software Configuration Management System.
 232        Copyright 1995-2011 Perforce Software.  All rights reserved.
 233        Rev. P4/NTX86/2011.1/393975 (2011/12/16).
 234     """
 235     global _p4_version_string
 236     if not _p4_version_string:
 237         a = p4_read_pipe_lines(["-V"])
 238         _p4_version_string = a[-1].rstrip()
 239     return _p4_version_string
 240
 241 def p4_integrate(src, dest):
 242     p4_system(["integrate", "-Dt", wildcard_encode(src), wildcard_encode(dest)])
 243
 244 def p4_sync(f, *options):
 245     p4_system(["sync"] + list(options) + [wildcard_encode(f)])
 246
 247 def p4_add(f):
 248     # forcibly add file names with wildcards
 249     if wildcard_present(f):
 250         p4_system(["add", "-f", f])
 251     else:
 252         p4_system(["add", f])
 253
 254 def p4_delete(f):
 255     p4_system(["delete", wildcard_encode(f)])
 256
 257 def p4_edit(f, *options):
 258     p4_system(["edit"] + list(options) + [wildcard_encode(f)])
 259
 260 def p4_revert(f):
 261     p4_system(["revert", wildcard_encode(f)])
 262
 263 def p4_reopen(type, f):
 264     p4_system(["reopen", "-t", type, wildcard_encode(f)])
 265
 266 def p4_move(src, dest):
 267     p4_system(["move", "-k", wildcard_encode(src), wildcard_encode(dest)])
 268
 269 def p4_last_change():
 270     results = p4CmdList(["changes", "-m", "1"])
 271     return int(results[0]['change'])
 272
 273 def p4_describe(change):
 274     """Make sure it returns a valid result by checking for
 275        the presence of field "time".  Return a dict of the
 276        results."""
 277
 278     ds = p4CmdList(["describe", "-s", str(change)])
 279     if len(ds) != 1:
 280         die("p4 describe -s %d did not return 1 result: %s" % (change, str(ds)))
 281
 282     d = ds[0]
 283
 284     if "p4ExitCode" in d:
 285         die("p4 describe -s %d exited with %d: %s" % (change, d["p4ExitCode"],
 286                                                       str(d)))
 287     if "code" in d:
 288         if d["code"] == "error":
 289             die("p4 describe -s %d returned error code: %s" % (change, str(d)))
 290
 291     if "time" not in d:
 292         die("p4 describe -s %d returned no \"time\": %s" % (change, str(d)))
 293
 294     return d
 295
 296 #
 297 # Canonicalize the p4 type and return a tuple of the
 298 # base type, plus any modifiers.  See "p4 help filetypes"
 299 # for a list and explanation.
 300 #
 301 def split_p4_type(p4type):
 302
 303     p4_filetypes_historical = {
 304         "ctempobj": "binary+Sw",
 305         "ctext": "text+C",
 306         "cxtext": "text+Cx",
 307         "ktext": "text+k",
 308         "kxtext": "text+kx",
 309         "ltext": "text+F",
 310         "tempobj": "binary+FSw",
 311         "ubinary": "binary+F",
 312         "uresource": "resource+F",
 313         "uxbinary": "binary+Fx",
 314         "xbinary": "binary+x",
 315         "xltext": "text+Fx",
 316         "xtempobj": "binary+Swx",
 317         "xtext": "text+x",
 318         "xunicode": "unicode+x",
 319         "xutf16": "utf16+x",
 320     }
 321     if p4type in p4_filetypes_historical:
 322         p4type = p4_filetypes_historical[p4type]
 323     mods = ""
 324     s = p4type.split("+")
 325     base = s[0]
 326     mods = ""
 327     if len(s) > 1:
 328         mods = s[1]
 329     return (base, mods)
 330
 331 #
 332 # return the raw p4 type of a file (text, text+ko, etc)
 333 #
 334 def p4_type(f):
 335     results = p4CmdList(["fstat", "-T", "headType", wildcard_encode(f)])
 336     return results[0]['headType']
 337
 338 #
 339 # Given a type base and modifier, return a regexp matching
 340 # the keywords that can be expanded in the file
 341 #
 342 def p4_keywords_regexp_for_type(base, type_mods):
 343     if base in ("text", "unicode", "binary"):
 344         kwords = None
 345         if "ko" in type_mods:
 346             kwords = 'Id|Header'
 347         elif "k" in type_mods:
 348             kwords = 'Id|Header|Author|Date|DateTime|Change|File|Revision'
 349         else:
 350             return None
 351         pattern = r"""
 352             \$              # Starts with a dollar, followed by...
 353             (%s)            # one of the keywords, followed by...
 354             (:[^$\n]+)?     # possibly an old expansion, followed by...
 355             \$              # another dollar
 356             """ % kwords
 357         return pattern
 358     else:
 359         return None
 360
 361 #
 362 # Given a file, return a regexp matching the possible
 363 # RCS keywords that will be expanded, or None for files
 364 # with kw expansion turned off.
 365 #
 366 def p4_keywords_regexp_for_file(file):
 367     if not os.path.exists(file):
 368         return None
 369     else:
 370         (type_base, type_mods) = split_p4_type(p4_type(file))
 371         return p4_keywords_regexp_for_type(type_base, type_mods)
 372
 373 def setP4ExecBit(file, mode):
 374     # Reopens an already open file and changes the execute bit to match
 375     # the execute bit setting in the passed in mode.
 376
 377     p4Type = "+x"
 378
 379     if not isModeExec(mode):
 380         p4Type = getP4OpenedType(file)
 381         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 382         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 383         if p4Type[-1] == "+":
 384             p4Type = p4Type[0:-1]
 385
 386     p4_reopen(p4Type, file)
 387
 388 def getP4OpenedType(file):
 389     # Returns the perforce file type for the given file.
 390
 391     result = p4_read_pipe(["opened", wildcard_encode(file)])
 392     match = re.match(".*\((.+)\)( \*exclusive\*)?\r?$", result)
 393     if match:
 394         return match.group(1)
 395     else:
 396         die("Could not determine file type for %s (result: '%s')" % (file, result))
 397
 398 # Return the set of all p4 labels
 399 def getP4Labels(depotPaths):
 400     labels = set()
 401     if isinstance(depotPaths,basestring):
 402         depotPaths = [depotPaths]
 403
 404     for l in p4CmdList(["labels"] + ["%s..." % p for p in depotPaths]):
 405         label = l['label']
 406         labels.add(label)
 407
 408     return labels
 409
 410 # Return the set of all git tags
 411 def getGitTags():
 412     gitTags = set()
 413     for line in read_pipe_lines(["git", "tag"]):
 414         tag = line.strip()
 415         gitTags.add(tag)
 416     return gitTags
 417
 418 def diffTreePattern():
 419     # This is a simple generator for the diff tree regex pattern. This could be
 420     # a class variable if this and parseDiffTreeEntry were a part of a class.
 421     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 422     while True:
 423         yield pattern
 424
 425 def parseDiffTreeEntry(entry):
 426     """Parses a single diff tree entry into its component elements.
 427
 428     See git-diff-tree(1) manpage for details about the format of the diff
 429     output. This method returns a dictionary with the following elements:
 430
 431     src_mode - The mode of the source file
 432     dst_mode - The mode of the destination file
 433     src_sha1 - The sha1 for the source file
 434     dst_sha1 - The sha1 fr the destination file
 435     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 436     status_score - The score for the status (applicable for 'C' and 'R'
 437                    statuses). This is None if there is no score.
 438     src - The path for the source file.
 439     dst - The path for the destination file. This is only present for
 440           copy or renames. If it is not present, this is None.
 441
 442     If the pattern is not matched, None is returned."""
 443
 444     match = diffTreePattern().next().match(entry)
 445     if match:
 446         return {
 447             'src_mode': match.group(1),
 448             'dst_mode': match.group(2),
 449             'src_sha1': match.group(3),
 450             'dst_sha1': match.group(4),
 451             'status': match.group(5),
 452             'status_score': match.group(6),
 453             'src': match.group(7),
 454             'dst': match.group(10)
 455         }
 456     return None
 457
 458 def isModeExec(mode):
 459     # Returns True if the given git mode represents an executable file,
 460     # otherwise False.
 461     return mode[-3:] == "755"
 462
 463 def isModeExecChanged(src_mode, dst_mode):
 464     return isModeExec(src_mode) != isModeExec(dst_mode)
 465
 466 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 467
 468     if isinstance(cmd,basestring):
 469         cmd = "-G " + cmd
 470         expand = True
 471     else:
 472         cmd = ["-G"] + cmd
 473         expand = False
 474
 475     cmd = p4_build_cmd(cmd)
 476     if verbose:
 477         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 478
 479     # Use a temporary file to avoid deadlocks without
 480     # subprocess.communicate(), which would put another copy
 481     # of stdout into memory.
 482     stdin_file = None
 483     if stdin is not None:
 484         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 485         if isinstance(stdin,basestring):
 486             stdin_file.write(stdin)
 487         else:
 488             for i in stdin:
 489                 stdin_file.write(i + '\n')
 490         stdin_file.flush()
 491         stdin_file.seek(0)
 492
 493     p4 = subprocess.Popen(cmd,
 494                           shell=expand,
 495                           stdin=stdin_file,
 496                           stdout=subprocess.PIPE)
 497
 498     result = []
 499     try:
 500         while True:
 501             entry = marshal.load(p4.stdout)
 502             if cb is not None:
 503                 cb(entry)
 504             else:
 505                 result.append(entry)
 506     except EOFError:
 507         pass
 508     exitCode = p4.wait()
 509     if exitCode != 0:
 510         entry = {}
 511         entry["p4ExitCode"] = exitCode
 512         result.append(entry)
 513
 514     return result
 515
 516 def p4Cmd(cmd):
 517     list = p4CmdList(cmd)
 518     result = {}
 519     for entry in list:
 520         result.update(entry)
 521     return result;
 522
 523 def p4Where(depotPath):
 524     if not depotPath.endswith("/"):
 525         depotPath += "/"
 526     depotPathLong = depotPath + "..."
 527     outputList = p4CmdList(["where", depotPathLong])
 528     output = None
 529     for entry in outputList:
 530         if "depotFile" in entry:
 531             # Search for the base client side depot path, as long as it starts with the branch's P4 path.
 532             # The base path always ends with "/...".
 533             if entry["depotFile"].find(depotPath) == 0 and entry["depotFile"][-4:] == "/...":
 534                 output = entry
 535                 break
 536         elif "data" in entry:
 537             data = entry.get("data")
 538             space = data.find(" ")
 539             if data[:space] == depotPath:
 540                 output = entry
 541                 break
 542     if output == None:
 543         return ""
 544     if output["code"] == "error":
 545         return ""
 546     clientPath = ""
 547     if "path" in output:
 548         clientPath = output.get("path")
 549     elif "data" in output:
 550         data = output.get("data")
 551         lastSpace = data.rfind(" ")
 552         clientPath = data[lastSpace + 1:]
 553
 554     if clientPath.endswith("..."):
 555         clientPath = clientPath[:-3]
 556     return clientPath
 557
 558 def currentGitBranch():
 559     retcode = system(["git", "symbolic-ref", "-q", "HEAD"], ignore_error=True)
 560     if retcode != 0:
 561         # on a detached head
 562         return None
 563     else:
 564         return read_pipe(["git", "name-rev", "HEAD"]).split(" ")[1].strip()
 565
 566 def isValidGitDir(path):
 567     if (os.path.exists(path + "/HEAD")
 568         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 569         return True;
 570     return False
 571
 572 def parseRevision(ref):
 573     return read_pipe("git rev-parse %s" % ref).strip()
 574
 575 def branchExists(ref):
 576     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 577                      ignore_error=True)
 578     return len(rev) > 0
 579
 580 def extractLogMessageFromGitCommit(commit):
 581     logMessage = ""
 582
 583     ## fixme: title is first line of commit, not 1st paragraph.
 584     foundTitle = False
 585     for log in read_pipe_lines("git cat-file commit %s" % commit):
 586        if not foundTitle:
 587            if len(log) == 1:
 588                foundTitle = True
 589            continue
 590
 591        logMessage += log
 592     return logMessage
 593
 594 def extractSettingsGitLog(log):
 595     values = {}
 596     for line in log.split("\n"):
 597         line = line.strip()
 598         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 599         if not m:
 600             continue
 601
 602         assignments = m.group(1).split (':')
 603         for a in assignments:
 604             vals = a.split ('=')
 605             key = vals[0].strip()
 606             val = ('='.join (vals[1:])).strip()
 607             if val.endswith ('\"') and val.startswith('"'):
 608                 val = val[1:-1]
 609
 610             values[key] = val
 611
 612     paths = values.get("depot-paths")
 613     if not paths:
 614         paths = values.get("depot-path")
 615     if paths:
 616         values['depot-paths'] = paths.split(',')
 617     return values
 618
 619 def gitBranchExists(branch):
 620     proc = subprocess.Popen(["git", "rev-parse", branch],
 621                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 622     return proc.wait() == 0;
 623
 624 _gitConfig = {}
 625
 626 def gitConfig(key, typeSpecifier=None):
 627     if not _gitConfig.has_key(key):
 628         cmd = [ "git", "config" ]
 629         if typeSpecifier:
 630             cmd += [ typeSpecifier ]
 631         cmd += [ key ]
 632         s = read_pipe(cmd, ignore_error=True)
 633         _gitConfig[key] = s.strip()
 634     return _gitConfig[key]
 635
 636 def gitConfigBool(key):
 637     """Return a bool, using git config --bool.  It is True only if the
 638        variable is set to true, and False if set to false or not present
 639        in the config."""
 640
 641     if not _gitConfig.has_key(key):
 642         _gitConfig[key] = gitConfig(key, '--bool') == "true"
 643     return _gitConfig[key]
 644
 645 def gitConfigInt(key):
 646     if not _gitConfig.has_key(key):
 647         cmd = [ "git", "config", "--int", key ]
 648         s = read_pipe(cmd, ignore_error=True)
 649         v = s.strip()
 650         try:
 651             _gitConfig[key] = int(gitConfig(key, '--int'))
 652         except ValueError:
 653             _gitConfig[key] = None
 654     return _gitConfig[key]
 655
 656 def gitConfigList(key):
 657     if not _gitConfig.has_key(key):
 658         s = read_pipe(["git", "config", "--get-all", key], ignore_error=True)
 659         _gitConfig[key] = s.strip().split(os.linesep)
 660         if _gitConfig[key] == ['']:
 661             _gitConfig[key] = []
 662     return _gitConfig[key]
 663
 664 def p4BranchesInGit(branchesAreInRemotes=True):
 665     """Find all the branches whose names start with "p4/", looking
 666        in remotes or heads as specified by the argument.  Return
 667        a dictionary of { branch: revision } for each one found.
 668        The branch names are the short names, without any
 669        "p4/" prefix."""
 670
 671     branches = {}
 672
 673     cmdline = "git rev-parse --symbolic "
 674     if branchesAreInRemotes:
 675         cmdline += "--remotes"
 676     else:
 677         cmdline += "--branches"
 678
 679     for line in read_pipe_lines(cmdline):
 680         line = line.strip()
 681
 682         # only import to p4/
 683         if not line.startswith('p4/'):
 684             continue
 685         # special symbolic ref to p4/master
 686         if line == "p4/HEAD":
 687             continue
 688
 689         # strip off p4/ prefix
 690         branch = line[len("p4/"):]
 691
 692         branches[branch] = parseRevision(line)
 693
 694     return branches
 695
 696 def branch_exists(branch):
 697     """Make sure that the given ref name really exists."""
 698
 699     cmd = [ "git", "rev-parse", "--symbolic", "--verify", branch ]
 700     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 701     out, _ = p.communicate()
 702     if p.returncode:
 703         return False
 704     # expect exactly one line of output: the branch name
 705     return out.rstrip() == branch
 706
 707 def findUpstreamBranchPoint(head = "HEAD"):
 708     branches = p4BranchesInGit()
 709     # map from depot-path to branch name
 710     branchByDepotPath = {}
 711     for branch in branches.keys():
 712         tip = branches[branch]
 713         log = extractLogMessageFromGitCommit(tip)
 714         settings = extractSettingsGitLog(log)
 715         if settings.has_key("depot-paths"):
 716             paths = ",".join(settings["depot-paths"])
 717             branchByDepotPath[paths] = "remotes/p4/" + branch
 718
 719     settings = None
 720     parent = 0
 721     while parent < 65535:
 722         commit = head + "~%s" % parent
 723         log = extractLogMessageFromGitCommit(commit)
 724         settings = extractSettingsGitLog(log)
 725         if settings.has_key("depot-paths"):
 726             paths = ",".join(settings["depot-paths"])
 727             if branchByDepotPath.has_key(paths):
 728                 return [branchByDepotPath[paths], settings]
 729
 730         parent = parent + 1
 731
 732     return ["", settings]
 733
 734 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 735     if not silent:
 736         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 737                % localRefPrefix)
 738
 739     originPrefix = "origin/p4/"
 740
 741     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 742         line = line.strip()
 743         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 744             continue
 745
 746         headName = line[len(originPrefix):]
 747         remoteHead = localRefPrefix + headName
 748         originHead = line
 749
 750         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 751         if (not original.has_key('depot-paths')
 752             or not original.has_key('change')):
 753             continue
 754
 755         update = False
 756         if not gitBranchExists(remoteHead):
 757             if verbose:
 758                 print "creating %s" % remoteHead
 759             update = True
 760         else:
 761             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 762             if settings.has_key('change') > 0:
 763                 if settings['depot-paths'] == original['depot-paths']:
 764                     originP4Change = int(original['change'])
 765                     p4Change = int(settings['change'])
 766                     if originP4Change > p4Change:
 767                         print ("%s (%s) is newer than %s (%s). "
 768                                "Updating p4 branch from origin."
 769                                % (originHead, originP4Change,
 770                                   remoteHead, p4Change))
 771                         update = True
 772                 else:
 773                     print ("Ignoring: %s was imported from %s while "
 774                            "%s was imported from %s"
 775                            % (originHead, ','.join(original['depot-paths']),
 776                               remoteHead, ','.join(settings['depot-paths'])))
 777
 778         if update:
 779             system("git update-ref %s %s" % (remoteHead, originHead))
 780
 781 def originP4BranchesExist():
 782         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 783
 784
 785 def p4ParseNumericChangeRange(parts):
 786     changeStart = int(parts[0][1:])
 787     if parts[1] == '#head':
 788         changeEnd = p4_last_change()
 789     else:
 790         changeEnd = int(parts[1])
 791
 792     return (changeStart, changeEnd)
 793
 794 def chooseBlockSize(blockSize):
 795     if blockSize:
 796         return blockSize
 797     else:
 798         return defaultBlockSize
 799
 800 def p4ChangesForPaths(depotPaths, changeRange, requestedBlockSize):
 801     assert depotPaths
 802
 803     # Parse the change range into start and end. Try to find integer
 804     # revision ranges as these can be broken up into blocks to avoid
 805     # hitting server-side limits (maxrows, maxscanresults). But if
 806     # that doesn't work, fall back to using the raw revision specifier
 807     # strings, without using block mode.
 808
 809     if changeRange is None or changeRange == '':
 810         changeStart = 1
 811         changeEnd = p4_last_change()
 812         block_size = chooseBlockSize(requestedBlockSize)
 813     else:
 814         parts = changeRange.split(',')
 815         assert len(parts) == 2
 816         try:
 817             (changeStart, changeEnd) = p4ParseNumericChangeRange(parts)
 818             block_size = chooseBlockSize(requestedBlockSize)
 819         except:
 820             changeStart = parts[0][1:]
 821             changeEnd = parts[1]
 822             if requestedBlockSize:
 823                 die("cannot use --changes-block-size with non-numeric revisions")
 824             block_size = None
 825
 826     changes = set()
 827
 828     # Retrieve changes a block at a time, to prevent running
 829     # into a MaxResults/MaxScanRows error from the server.
 830
 831     while True:
 832         cmd = ['changes']
 833
 834         if block_size:
 835             end = min(changeEnd, changeStart + block_size)
 836             revisionRange = "%d,%d" % (changeStart, end)
 837         else:
 838             revisionRange = "%s,%s" % (changeStart, changeEnd)
 839
 840         for p in depotPaths:
 841             cmd += ["%s...@%s" % (p, revisionRange)]
 842
 843         # Insert changes in chronological order
 844         for line in reversed(p4_read_pipe_lines(cmd)):
 845             changes.add(int(line.split(" ")[1]))
 846
 847         if not block_size:
 848             break
 849
 850         if end >= changeEnd:
 851             break
 852
 853         changeStart = end + 1
 854
 855     changes = sorted(changes)
 856     return changes
 857
 858 def p4PathStartsWith(path, prefix):
 859     # This method tries to remedy a potential mixed-case issue:
 860     #
 861     # If UserA adds  //depot/DirA/file1
 862     # and UserB adds //depot/dira/file2
 863     #
 864     # we may or may not have a problem. If you have core.ignorecase=true,
 865     # we treat DirA and dira as the same directory
 866     if gitConfigBool("core.ignorecase"):
 867         return path.lower().startswith(prefix.lower())
 868     return path.startswith(prefix)
 869
 870 def getClientSpec():
 871     """Look at the p4 client spec, create a View() object that contains
 872        all the mappings, and return it."""
 873
 874     specList = p4CmdList("client -o")
 875     if len(specList) != 1:
 876         die('Output from "client -o" is %d lines, expecting 1' %
 877             len(specList))
 878
 879     # dictionary of all client parameters
 880     entry = specList[0]
 881
 882     # the //client/ name
 883     client_name = entry["Client"]
 884
 885     # just the keys that start with "View"
 886     view_keys = [ k for k in entry.keys() if k.startswith("View") ]
 887
 888     # hold this new View
 889     view = View(client_name)
 890
 891     # append the lines, in order, to the view
 892     for view_num in range(len(view_keys)):
 893         k = "View%d" % view_num
 894         if k not in view_keys:
 895             die("Expected view key %s missing" % k)
 896         view.append(entry[k])
 897
 898     return view
 899
 900 def getClientRoot():
 901     """Grab the client directory."""
 902
 903     output = p4CmdList("client -o")
 904     if len(output) != 1:
 905         die('Output from "client -o" is %d lines, expecting 1' % len(output))
 906
 907     entry = output[0]
 908     if "Root" not in entry:
 909         die('Client has no "Root"')
 910
 911     return entry["Root"]
 912
 913 #
 914 # P4 wildcards are not allowed in filenames.  P4 complains
 915 # if you simply add them, but you can force it with "-f", in
 916 # which case it translates them into %xx encoding internally.
 917 #
 918 def wildcard_decode(path):
 919     # Search for and fix just these four characters.  Do % last so
 920     # that fixing it does not inadvertently create new %-escapes.
 921     # Cannot have * in a filename in windows; untested as to
 922     # what p4 would do in such a case.
 923     if not platform.system() == "Windows":
 924         path = path.replace("%2A", "*")
 925     path = path.replace("%23", "#") \
 926                .replace("%40", "@") \
 927                .replace("%25", "%")
 928     return path
 929
 930 def wildcard_encode(path):
 931     # do % first to avoid double-encoding the %s introduced here
 932     path = path.replace("%", "%25") \
 933                .replace("*", "%2A") \
 934                .replace("#", "%23") \
 935                .replace("@", "%40")
 936     return path
 937
 938 def wildcard_present(path):
 939     m = re.search("[*#@%]", path)
 940     return m is not None
 941
 942 class LargeFileSystem(object):
 943     """Base class for large file system support."""
 944
 945     def __init__(self, writeToGitStream):
 946         self.largeFiles = set()
 947         self.writeToGitStream = writeToGitStream
 948
 949     def generatePointer(self, cloneDestination, contentFile):
 950         """Return the content of a pointer file that is stored in Git instead of
 951            the actual content."""
 952         assert False, "Method 'generatePointer' required in " + self.__class__.__name__
 953
 954     def pushFile(self, localLargeFile):
 955         """Push the actual content which is not stored in the Git repository to
 956            a server."""
 957         assert False, "Method 'pushFile' required in " + self.__class__.__name__
 958
 959     def hasLargeFileExtension(self, relPath):
 960         return reduce(
 961             lambda a, b: a or b,
 962             [relPath.endswith('.' + e) for e in gitConfigList('git-p4.largeFileExtensions')],
 963             False
 964         )
 965
 966     def generateTempFile(self, contents):
 967         contentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 968         for d in contents:
 969             contentFile.write(d)
 970         contentFile.close()
 971         return contentFile.name
 972
 973     def exceedsLargeFileThreshold(self, relPath, contents):
 974         if gitConfigInt('git-p4.largeFileThreshold'):
 975             contentsSize = sum(len(d) for d in contents)
 976             if contentsSize > gitConfigInt('git-p4.largeFileThreshold'):
 977                 return True
 978         if gitConfigInt('git-p4.largeFileCompressedThreshold'):
 979             contentsSize = sum(len(d) for d in contents)
 980             if contentsSize <= gitConfigInt('git-p4.largeFileCompressedThreshold'):
 981                 return False
 982             contentTempFile = self.generateTempFile(contents)
 983             compressedContentFile = tempfile.NamedTemporaryFile(prefix='git-p4-large-file', delete=False)
 984             zf = zipfile.ZipFile(compressedContentFile.name, mode='w')
 985             zf.write(contentTempFile, compress_type=zipfile.ZIP_DEFLATED)
 986             zf.close()
 987             compressedContentsSize = zf.infolist()[0].compress_size
 988             os.remove(contentTempFile)
 989             os.remove(compressedContentFile.name)
 990             if compressedContentsSize > gitConfigInt('git-p4.largeFileCompressedThreshold'):
 991                 return True
 992         return False
 993
 994     def addLargeFile(self, relPath):
 995         self.largeFiles.add(relPath)
 996
 997     def removeLargeFile(self, relPath):
 998         self.largeFiles.remove(relPath)
 999
1000     def isLargeFile(self, relPath):
1001         return relPath in self.largeFiles
1002
1003     def processContent(self, git_mode, relPath, contents):
1004         """Processes the content of git fast import. This method decides if a
1005            file is stored in the large file system and handles all necessary
1006            steps."""
1007         if self.exceedsLargeFileThreshold(relPath, contents) or self.hasLargeFileExtension(relPath):
1008             contentTempFile = self.generateTempFile(contents)
1009             (pointer_git_mode, contents, localLargeFile) = self.generatePointer(contentTempFile)
1010             if pointer_git_mode:
1011                 git_mode = pointer_git_mode
1012             if localLargeFile:
1013                 # Move temp file to final location in large file system
1014                 largeFileDir = os.path.dirname(localLargeFile)
1015                 if not os.path.isdir(largeFileDir):
1016                     os.makedirs(largeFileDir)
1017                 shutil.move(contentTempFile, localLargeFile)
1018                 self.addLargeFile(relPath)
1019                 if gitConfigBool('git-p4.largeFilePush'):
1020                     self.pushFile(localLargeFile)
1021                 if verbose:
1022                     sys.stderr.write("%s moved to large file system (%s)\n" % (relPath, localLargeFile))
1023         return (git_mode, contents)
1024
1025 class MockLFS(LargeFileSystem):
1026     """Mock large file system for testing."""
1027
1028     def generatePointer(self, contentFile):
1029         """The pointer content is the original content prefixed with "pointer-".
1030            The local filename of the large file storage is derived from the file content.
1031            """
1032         with open(contentFile, 'r') as f:
1033             content = next(f)
1034             gitMode = '100644'
1035             pointerContents = 'pointer-' + content
1036             localLargeFile = os.path.join(os.getcwd(), '.git', 'mock-storage', 'local', content[:-1])
1037             return (gitMode, pointerContents, localLargeFile)
1038
1039     def pushFile(self, localLargeFile):
1040         """The remote filename of the large file storage is the same as the local
1041            one but in a different directory.
1042            """
1043         remotePath = os.path.join(os.path.dirname(localLargeFile), '..', 'remote')
1044         if not os.path.exists(remotePath):
1045             os.makedirs(remotePath)
1046         shutil.copyfile(localLargeFile, os.path.join(remotePath, os.path.basename(localLargeFile)))
1047
1048 class GitLFS(LargeFileSystem):
1049     """Git LFS as backend for the git-p4 large file system.
1050        See https://git-lfs.github.com/ for details."""
1051
1052     def __init__(self, *args):
1053         LargeFileSystem.__init__(self, *args)
1054         self.baseGitAttributes = []
1055
1056     def generatePointer(self, contentFile):
1057         """Generate a Git LFS pointer for the content. Return LFS Pointer file
1058            mode and content which is stored in the Git repository instead of
1059            the actual content. Return also the new location of the actual
1060            content.
1061            """
1062         if os.path.getsize(contentFile) == 0:
1063             return (None, '', None)
1064
1065         pointerProcess = subprocess.Popen(
1066             ['git', 'lfs', 'pointer', '--file=' + contentFile],
1067             stdout=subprocess.PIPE
1068         )
1069         pointerFile = pointerProcess.stdout.read()
1070         if pointerProcess.wait():
1071             os.remove(contentFile)
1072             die('git-lfs pointer command failed. Did you install the extension?')
1073
1074         # Git LFS removed the preamble in the output of the 'pointer' command
1075         # starting from version 1.2.0. Check for the preamble here to support
1076         # earlier versions.
1077         # c.f. https://github.com/github/git-lfs/commit/da2935d9a739592bc775c98d8ef4df9c72ea3b43
1078         if pointerFile.startswith('Git LFS pointer for'):
1079             pointerFile = re.sub(r'Git LFS pointer for.*\n\n', '', pointerFile)
1080
1081         oid = re.search(r'^oid \w+:(\w+)', pointerFile, re.MULTILINE).group(1)
1082         localLargeFile = os.path.join(
1083             os.getcwd(),
1084             '.git', 'lfs', 'objects', oid[:2], oid[2:4],
1085             oid,
1086         )
1087         # LFS Spec states that pointer files should not have the executable bit set.
1088         gitMode = '100644'
1089         return (gitMode, pointerFile, localLargeFile)
1090
1091     def pushFile(self, localLargeFile):
1092         uploadProcess = subprocess.Popen(
1093             ['git', 'lfs', 'push', '--object-id', 'origin', os.path.basename(localLargeFile)]
1094         )
1095         if uploadProcess.wait():
1096             die('git-lfs push command failed. Did you define a remote?')
1097
1098     def generateGitAttributes(self):
1099         return (
1100             self.baseGitAttributes +
1101             [
1102                 '\n',
1103                 '#\n',
1104                 '# Git LFS (see https://git-lfs.github.com/)\n',
1105                 '#\n',
1106             ] +
1107             ['*.' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1108                 for f in sorted(gitConfigList('git-p4.largeFileExtensions'))
1109             ] +
1110             ['/' + f.replace(' ', '[[:space:]]') + ' filter=lfs -text\n'
1111                 for f in sorted(self.largeFiles) if not self.hasLargeFileExtension(f)
1112             ]
1113         )
1114
1115     def addLargeFile(self, relPath):
1116         LargeFileSystem.addLargeFile(self, relPath)
1117         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1118
1119     def removeLargeFile(self, relPath):
1120         LargeFileSystem.removeLargeFile(self, relPath)
1121         self.writeToGitStream('100644', '.gitattributes', self.generateGitAttributes())
1122
1123     def processContent(self, git_mode, relPath, contents):
1124         if relPath == '.gitattributes':
1125             self.baseGitAttributes = contents
1126             return (git_mode, self.generateGitAttributes())
1127         else:
1128             return LargeFileSystem.processContent(self, git_mode, relPath, contents)
1129
1130 class Command:
1131     def __init__(self):
1132         self.usage = "usage: %prog [options]"
1133         self.needsGit = True
1134         self.verbose = False
1135
1136 class P4UserMap:
1137     def __init__(self):
1138         self.userMapFromPerforceServer = False
1139         self.myP4UserId = None
1140
1141     def p4UserId(self):
1142         if self.myP4UserId:
1143             return self.myP4UserId
1144
1145         results = p4CmdList("user -o")
1146         for r in results:
1147             if r.has_key('User'):
1148                 self.myP4UserId = r['User']
1149                 return r['User']
1150         die("Could not find your p4 user id")
1151
1152     def p4UserIsMe(self, p4User):
1153         # return True if the given p4 user is actually me
1154         me = self.p4UserId()
1155         if not p4User or p4User != me:
1156             return False
1157         else:
1158             return True
1159
1160     def getUserCacheFilename(self):
1161         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1162         return home + "/.gitp4-usercache.txt"
1163
1164     def getUserMapFromPerforceServer(self):
1165         if self.userMapFromPerforceServer:
1166             return
1167         self.users = {}
1168         self.emails = {}
1169
1170         for output in p4CmdList("users"):
1171             if not output.has_key("User"):
1172                 continue
1173             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1174             self.emails[output["Email"]] = output["User"]
1175
1176         mapUserConfigRegex = re.compile(r"^\s*(\S+)\s*=\s*(.+)\s*<(\S+)>\s*$", re.VERBOSE)
1177         for mapUserConfig in gitConfigList("git-p4.mapUser"):
1178             mapUser = mapUserConfigRegex.findall(mapUserConfig)
1179             if mapUser and len(mapUser[0]) == 3:
1180                 user = mapUser[0][0]
1181                 fullname = mapUser[0][1]
1182                 email = mapUser[0][2]
1183                 self.users[user] = fullname + " <" + email + ">"
1184                 self.emails[email] = user
1185
1186         s = ''
1187         for (key, val) in self.users.items():
1188             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1189
1190         open(self.getUserCacheFilename(), "wb").write(s)
1191         self.userMapFromPerforceServer = True
1192
1193     def loadUserMapFromCache(self):
1194         self.users = {}
1195         self.userMapFromPerforceServer = False
1196         try:
1197             cache = open(self.getUserCacheFilename(), "rb")
1198             lines = cache.readlines()
1199             cache.close()
1200             for line in lines:
1201                 entry = line.strip().split("\t")
1202                 self.users[entry[0]] = entry[1]
1203         except IOError:
1204             self.getUserMapFromPerforceServer()
1205
1206 class P4Debug(Command):
1207     def __init__(self):
1208         Command.__init__(self)
1209         self.options = []
1210         self.description = "A tool to debug the output of p4 -G."
1211         self.needsGit = False
1212
1213     def run(self, args):
1214         j = 0
1215         for output in p4CmdList(args):
1216             print 'Element: %d' % j
1217             j += 1
1218             print output
1219         return True
1220
1221 class P4RollBack(Command):
1222     def __init__(self):
1223         Command.__init__(self)
1224         self.options = [
1225             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
1226         ]
1227         self.description = "A tool to debug the multi-branch import. Don't use :)"
1228         self.rollbackLocalBranches = False
1229
1230     def run(self, args):
1231         if len(args) != 1:
1232             return False
1233         maxChange = int(args[0])
1234
1235         if "p4ExitCode" in p4Cmd("changes -m 1"):
1236             die("Problems executing p4");
1237
1238         if self.rollbackLocalBranches:
1239             refPrefix = "refs/heads/"
1240             lines = read_pipe_lines("git rev-parse --symbolic --branches")
1241         else:
1242             refPrefix = "refs/remotes/"
1243             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
1244
1245         for line in lines:
1246             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
1247                 line = line.strip()
1248                 ref = refPrefix + line
1249                 log = extractLogMessageFromGitCommit(ref)
1250                 settings = extractSettingsGitLog(log)
1251
1252                 depotPaths = settings['depot-paths']
1253                 change = settings['change']
1254
1255                 changed = False
1256
1257                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
1258                                                            for p in depotPaths]))) == 0:
1259                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
1260                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
1261                     continue
1262
1263                 while change and int(change) > maxChange:
1264                     changed = True
1265                     if self.verbose:
1266                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
1267                     system("git update-ref %s \"%s^\"" % (ref, ref))
1268                     log = extractLogMessageFromGitCommit(ref)
1269                     settings =  extractSettingsGitLog(log)
1270
1271
1272                     depotPaths = settings['depot-paths']
1273                     change = settings['change']
1274
1275                 if changed:
1276                     print "%s rewound to %s" % (ref, change)
1277
1278         return True
1279
1280 class P4Submit(Command, P4UserMap):
1281
1282     conflict_behavior_choices = ("ask", "skip", "quit")
1283
1284     def __init__(self):
1285         Command.__init__(self)
1286         P4UserMap.__init__(self)
1287         self.options = [
1288                 optparse.make_option("--origin", dest="origin"),
1289                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
1290                 # preserve the user, requires relevant p4 permissions
1291                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
1292                 optparse.make_option("--export-labels", dest="exportLabels", action="store_true"),
1293                 optparse.make_option("--dry-run", "-n", dest="dry_run", action="store_true"),
1294                 optparse.make_option("--prepare-p4-only", dest="prepare_p4_only", action="store_true"),
1295                 optparse.make_option("--conflict", dest="conflict_behavior",
1296                                      choices=self.conflict_behavior_choices),
1297                 optparse.make_option("--branch", dest="branch"),
1298         ]
1299         self.description = "Submit changes from git to the perforce depot."
1300         self.usage += " [name of git branch to submit into perforce depot]"
1301         self.origin = ""
1302         self.detectRenames = False
1303         self.preserveUser = gitConfigBool("git-p4.preserveUser")
1304         self.dry_run = False
1305         self.prepare_p4_only = False
1306         self.conflict_behavior = None
1307         self.isWindows = (platform.system() == "Windows")
1308         self.exportLabels = False
1309         self.p4HasMoveCommand = p4_has_move_command()
1310         self.branch = None
1311
1312         if gitConfig('git-p4.largeFileSystem'):
1313             die("Large file system not supported for git-p4 submit command. Please remove it from config.")
1314
1315     def check(self):
1316         if len(p4CmdList("opened ...")) > 0:
1317             die("You have files opened with perforce! Close them before starting the sync.")
1318
1319     def separate_jobs_from_description(self, message):
1320         """Extract and return a possible Jobs field in the commit
1321            message.  It goes into a separate section in the p4 change
1322            specification.
1323
1324            A jobs line starts with "Jobs:" and looks like a new field
1325            in a form.  Values are white-space separated on the same
1326            line or on following lines that start with a tab.
1327
1328            This does not parse and extract the full git commit message
1329            like a p4 form.  It just sees the Jobs: line as a marker
1330            to pass everything from then on directly into the p4 form,
1331            but outside the description section.
1332
1333            Return a tuple (stripped log message, jobs string)."""
1334
1335         m = re.search(r'^Jobs:', message, re.MULTILINE)
1336         if m is None:
1337             return (message, None)
1338
1339         jobtext = message[m.start():]
1340         stripped_message = message[:m.start()].rstrip()
1341         return (stripped_message, jobtext)
1342
1343     def prepareLogMessage(self, template, message, jobs):
1344         """Edits the template returned from "p4 change -o" to insert
1345            the message in the Description field, and the jobs text in
1346            the Jobs field."""
1347         result = ""
1348
1349         inDescriptionSection = False
1350
1351         for line in template.split("\n"):
1352             if line.startswith("#"):
1353                 result += line + "\n"
1354                 continue
1355
1356             if inDescriptionSection:
1357                 if line.startswith("Files:") or line.startswith("Jobs:"):
1358                     inDescriptionSection = False
1359                     # insert Jobs section
1360                     if jobs:
1361                         result += jobs + "\n"
1362                 else:
1363                     continue
1364             else:
1365                 if line.startswith("Description:"):
1366                     inDescriptionSection = True
1367                     line += "\n"
1368                     for messageLine in message.split("\n"):
1369                         line += "\t" + messageLine + "\n"
1370
1371             result += line + "\n"
1372
1373         return result
1374
1375     def patchRCSKeywords(self, file, pattern):
1376         # Attempt to zap the RCS keywords in a p4 controlled file matching the given pattern
1377         (handle, outFileName) = tempfile.mkstemp(dir='.')
1378         try:
1379             outFile = os.fdopen(handle, "w+")
1380             inFile = open(file, "r")
1381             regexp = re.compile(pattern, re.VERBOSE)
1382             for line in inFile.readlines():
1383                 line = regexp.sub(r'$\1$', line)
1384                 outFile.write(line)
1385             inFile.close()
1386             outFile.close()
1387             # Forcibly overwrite the original file
1388             os.unlink(file)
1389             shutil.move(outFileName, file)
1390         except:
1391             # cleanup our temporary file
1392             os.unlink(outFileName)
1393             print "Failed to strip RCS keywords in %s" % file
1394             raise
1395
1396         print "Patched up RCS keywords in %s" % file
1397
1398     def p4UserForCommit(self,id):
1399         # Return the tuple (perforce user,git email) for a given git commit id
1400         self.getUserMapFromPerforceServer()
1401         gitEmail = read_pipe(["git", "log", "--max-count=1",
1402                               "--format=%ae", id])
1403         gitEmail = gitEmail.strip()
1404         if not self.emails.has_key(gitEmail):
1405             return (None,gitEmail)
1406         else:
1407             return (self.emails[gitEmail],gitEmail)
1408
1409     def checkValidP4Users(self,commits):
1410         # check if any git authors cannot be mapped to p4 users
1411         for id in commits:
1412             (user,email) = self.p4UserForCommit(id)
1413             if not user:
1414                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
1415                 if gitConfigBool("git-p4.allowMissingP4Users"):
1416                     print "%s" % msg
1417                 else:
1418                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
1419
1420     def lastP4Changelist(self):
1421         # Get back the last changelist number submitted in this client spec. This
1422         # then gets used to patch up the username in the change. If the same
1423         # client spec is being used by multiple processes then this might go
1424         # wrong.
1425         results = p4CmdList("client -o")        # find the current client
1426         client = None
1427         for r in results:
1428             if r.has_key('Client'):
1429                 client = r['Client']
1430                 break
1431         if not client:
1432             die("could not get client spec")
1433         results = p4CmdList(["changes", "-c", client, "-m", "1"])
1434         for r in results:
1435             if r.has_key('change'):
1436                 return r['change']
1437         die("Could not get changelist number for last submit - cannot patch up user details")
1438
1439     def modifyChangelistUser(self, changelist, newUser):
1440         # fixup the user field of a changelist after it has been submitted.
1441         changes = p4CmdList("change -o %s" % changelist)
1442         if len(changes) != 1:
1443             die("Bad output from p4 change modifying %s to user %s" %
1444                 (changelist, newUser))
1445
1446         c = changes[0]
1447         if c['User'] == newUser: return   # nothing to do
1448         c['User'] = newUser
1449         input = marshal.dumps(c)
1450
1451         result = p4CmdList("change -f -i", stdin=input)
1452         for r in result:
1453             if r.has_key('code'):
1454                 if r['code'] == 'error':
1455                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
1456             if r.has_key('data'):
1457                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
1458                 return
1459         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
1460
1461     def canChangeChangelists(self):
1462         # check to see if we have p4 admin or super-user permissions, either of
1463         # which are required to modify changelists.
1464         results = p4CmdList(["protects", self.depotPath])
1465         for r in results:
1466             if r.has_key('perm'):
1467                 if r['perm'] == 'admin':
1468                     return 1
1469                 if r['perm'] == 'super':
1470                     return 1
1471         return 0
1472
1473     def prepareSubmitTemplate(self):
1474         """Run "p4 change -o" to grab a change specification template.
1475            This does not use "p4 -G", as it is nice to keep the submission
1476            template in original order, since a human might edit it.
1477
1478            Remove lines in the Files section that show changes to files
1479            outside the depot path we're committing into."""
1480
1481         [upstream, settings] = findUpstreamBranchPoint()
1482
1483         template = ""
1484         inFilesSection = False
1485         for line in p4_read_pipe_lines(['change', '-o']):
1486             if line.endswith("\r\n"):
1487                 line = line[:-2] + "\n"
1488             if inFilesSection:
1489                 if line.startswith("\t"):
1490                     # path starts and ends with a tab
1491                     path = line[1:]
1492                     lastTab = path.rfind("\t")
1493                     if lastTab != -1:
1494                         path = path[:lastTab]
1495                         if settings.has_key('depot-paths'):
1496                             if not [p for p in settings['depot-paths']
1497                                     if p4PathStartsWith(path, p)]:
1498                                 continue
1499                         else:
1500                             if not p4PathStartsWith(path, self.depotPath):
1501                                 continue
1502                 else:
1503                     inFilesSection = False
1504             else:
1505                 if line.startswith("Files:"):
1506                     inFilesSection = True
1507
1508             template += line
1509
1510         return template
1511
1512     def edit_template(self, template_file):
1513         """Invoke the editor to let the user change the submission
1514            message.  Return true if okay to continue with the submit."""
1515
1516         # if configured to skip the editing part, just submit
1517         if gitConfigBool("git-p4.skipSubmitEdit"):
1518             return True
1519
1520         # look at the modification time, to check later if the user saved
1521         # the file
1522         mtime = os.stat(template_file).st_mtime
1523
1524         # invoke the editor
1525         if os.environ.has_key("P4EDITOR") and (os.environ.get("P4EDITOR") != ""):
1526             editor = os.environ.get("P4EDITOR")
1527         else:
1528             editor = read_pipe("git var GIT_EDITOR").strip()
1529         system(["sh", "-c", ('%s "$@"' % editor), editor, template_file])
1530
1531         # If the file was not saved, prompt to see if this patch should
1532         # be skipped.  But skip this verification step if configured so.
1533         if gitConfigBool("git-p4.skipSubmitEditCheck"):
1534             return True
1535
1536         # modification time updated means user saved the file
1537         if os.stat(template_file).st_mtime > mtime:
1538             return True
1539
1540         while True:
1541             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
1542             if response == 'y':
1543                 return True
1544             if response == 'n':
1545                 return False
1546
1547     def get_diff_description(self, editedFiles, filesToAdd, symlinks):
1548         # diff
1549         if os.environ.has_key("P4DIFF"):
1550             del(os.environ["P4DIFF"])
1551         diff = ""
1552         for editedFile in editedFiles:
1553             diff += p4_read_pipe(['diff', '-du',
1554                                   wildcard_encode(editedFile)])
1555
1556         # new file diff
1557         newdiff = ""
1558         for newFile in filesToAdd:
1559             newdiff += "==== new file ====\n"
1560             newdiff += "--- /dev/null\n"
1561             newdiff += "+++ %s\n" % newFile
1562
1563             is_link = os.path.islink(newFile)
1564             expect_link = newFile in symlinks
1565
1566             if is_link and expect_link:
1567                 newdiff += "+%s\n" % os.readlink(newFile)
1568             else:
1569                 f = open(newFile, "r")
1570                 for line in f.readlines():
1571                     newdiff += "+" + line
1572                 f.close()
1573
1574         return (diff + newdiff).replace('\r\n', '\n')
1575
1576     def applyCommit(self, id):
1577         """Apply one commit, return True if it succeeded."""
1578
1579         print "Applying", read_pipe(["git", "show", "-s",
1580                                      "--format=format:%h %s", id])
1581
1582         (p4User, gitEmail) = self.p4UserForCommit(id)
1583
1584         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (self.diffOpts, id, id))
1585         filesToAdd = set()
1586         filesToChangeType = set()
1587         filesToDelete = set()
1588         editedFiles = set()
1589         pureRenameCopy = set()
1590         symlinks = set()
1591         filesToChangeExecBit = {}
1592
1593         for line in diff:
1594             diff = parseDiffTreeEntry(line)
1595             modifier = diff['status']
1596             path = diff['src']
1597             if modifier == "M":
1598                 p4_edit(path)
1599                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1600                     filesToChangeExecBit[path] = diff['dst_mode']
1601                 editedFiles.add(path)
1602             elif modifier == "A":
1603                 filesToAdd.add(path)
1604                 filesToChangeExecBit[path] = diff['dst_mode']
1605                 if path in filesToDelete:
1606                     filesToDelete.remove(path)
1607
1608                 dst_mode = int(diff['dst_mode'], 8)
1609                 if dst_mode == 0120000:
1610                     symlinks.add(path)
1611
1612             elif modifier == "D":
1613                 filesToDelete.add(path)
1614                 if path in filesToAdd:
1615                     filesToAdd.remove(path)
1616             elif modifier == "C":
1617                 src, dest = diff['src'], diff['dst']
1618                 p4_integrate(src, dest)
1619                 pureRenameCopy.add(dest)
1620                 if diff['src_sha1'] != diff['dst_sha1']:
1621                     p4_edit(dest)
1622                     pureRenameCopy.discard(dest)
1623                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1624                     p4_edit(dest)
1625                     pureRenameCopy.discard(dest)
1626                     filesToChangeExecBit[dest] = diff['dst_mode']
1627                 if self.isWindows:
1628                     # turn off read-only attribute
1629                     os.chmod(dest, stat.S_IWRITE)
1630                 os.unlink(dest)
1631                 editedFiles.add(dest)
1632             elif modifier == "R":
1633                 src, dest = diff['src'], diff['dst']
1634                 if self.p4HasMoveCommand:
1635                     p4_edit(src)        # src must be open before move
1636                     p4_move(src, dest)  # opens for (move/delete, move/add)
1637                 else:
1638                     p4_integrate(src, dest)
1639                     if diff['src_sha1'] != diff['dst_sha1']:
1640                         p4_edit(dest)
1641                     else:
1642                         pureRenameCopy.add(dest)
1643                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
1644                     if not self.p4HasMoveCommand:
1645                         p4_edit(dest)   # with move: already open, writable
1646                     filesToChangeExecBit[dest] = diff['dst_mode']
1647                 if not self.p4HasMoveCommand:
1648                     if self.isWindows:
1649                         os.chmod(dest, stat.S_IWRITE)
1650                     os.unlink(dest)
1651                     filesToDelete.add(src)
1652                 editedFiles.add(dest)
1653             elif modifier == "T":
1654                 filesToChangeType.add(path)
1655             else:
1656                 die("unknown modifier %s for %s" % (modifier, path))
1657
1658         diffcmd = "git diff-tree --full-index -p \"%s\"" % (id)
1659         patchcmd = diffcmd + " | git apply "
1660         tryPatchCmd = patchcmd + "--check -"
1661         applyPatchCmd = patchcmd + "--check --apply -"
1662         patch_succeeded = True
1663
1664         if os.system(tryPatchCmd) != 0:
1665             fixed_rcs_keywords = False
1666             patch_succeeded = False
1667             print "Unfortunately applying the change failed!"
1668
1669             # Patch failed, maybe it's just RCS keyword woes. Look through
1670             # the patch to see if that's possible.
1671             if gitConfigBool("git-p4.attemptRCSCleanup"):
1672                 file = None
1673                 pattern = None
1674                 kwfiles = {}
1675                 for file in editedFiles | filesToDelete:
1676                     # did this file's delta contain RCS keywords?
1677                     pattern = p4_keywords_regexp_for_file(file)
1678
1679                     if pattern:
1680                         # this file is a possibility...look for RCS keywords.
1681                         regexp = re.compile(pattern, re.VERBOSE)
1682                         for line in read_pipe_lines(["git", "diff", "%s^..%s" % (id, id), file]):
1683                             if regexp.search(line):
1684                                 if verbose:
1685                                     print "got keyword match on %s in %s in %s" % (pattern, line, file)
1686                                 kwfiles[file] = pattern
1687                                 break
1688
1689                 for file in kwfiles:
1690                     if verbose:
1691                         print "zapping %s with %s" % (line,pattern)
1692                     # File is being deleted, so not open in p4.  Must
1693                     # disable the read-only bit on windows.
1694                     if self.isWindows and file not in editedFiles:
1695                         os.chmod(file, stat.S_IWRITE)
1696                     self.patchRCSKeywords(file, kwfiles[file])
1697                     fixed_rcs_keywords = True
1698
1699             if fixed_rcs_keywords:
1700                 print "Retrying the patch with RCS keywords cleaned up"
1701                 if os.system(tryPatchCmd) == 0:
1702                     patch_succeeded = True
1703
1704         if not patch_succeeded:
1705             for f in editedFiles:
1706                 p4_revert(f)
1707             return False
1708
1709         #
1710         # Apply the patch for real, and do add/delete/+x handling.
1711         #
1712         system(applyPatchCmd)
1713
1714         for f in filesToChangeType:
1715             p4_edit(f, "-t", "auto")
1716         for f in filesToAdd:
1717             p4_add(f)
1718         for f in filesToDelete:
1719             p4_revert(f)
1720             p4_delete(f)
1721
1722         # Set/clear executable bits
1723         for f in filesToChangeExecBit.keys():
1724             mode = filesToChangeExecBit[f]
1725             setP4ExecBit(f, mode)
1726
1727         #
1728         # Build p4 change description, starting with the contents
1729         # of the git commit message.
1730         #
1731         logMessage = extractLogMessageFromGitCommit(id)
1732         logMessage = logMessage.strip()
1733         (logMessage, jobs) = self.separate_jobs_from_description(logMessage)
1734
1735         template = self.prepareSubmitTemplate()
1736         submitTemplate = self.prepareLogMessage(template, logMessage, jobs)
1737
1738         if self.preserveUser:
1739            submitTemplate += "\n######## Actual user %s, modified after commit\n" % p4User
1740
1741         if self.checkAuthorship and not self.p4UserIsMe(p4User):
1742             submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1743             submitTemplate += "######## Use option --preserve-user to modify authorship.\n"
1744             submitTemplate += "######## Variable git-p4.skipUserNameCheck hides this message.\n"
1745
1746         separatorLine = "######## everything below this line is just the diff #######\n"
1747         if not self.prepare_p4_only:
1748             submitTemplate += separatorLine
1749             submitTemplate += self.get_diff_description(editedFiles, filesToAdd, symlinks)
1750
1751         (handle, fileName) = tempfile.mkstemp()
1752         tmpFile = os.fdopen(handle, "w+b")
1753         if self.isWindows:
1754             submitTemplate = submitTemplate.replace("\n", "\r\n")
1755         tmpFile.write(submitTemplate)
1756         tmpFile.close()
1757
1758         if self.prepare_p4_only:
1759             #
1760             # Leave the p4 tree prepared, and the submit template around
1761             # and let the user decide what to do next
1762             #
1763             print
1764             print "P4 workspace prepared for submission."
1765             print "To submit or revert, go to client workspace"
1766             print "  " + self.clientPath
1767             print
1768             print "To submit, use \"p4 submit\" to write a new description,"
1769             print "or \"p4 submit -i <%s\" to use the one prepared by" \
1770                   " \"git p4\"." % fileName
1771             print "You can delete the file \"%s\" when finished." % fileName
1772
1773             if self.preserveUser and p4User and not self.p4UserIsMe(p4User):
1774                 print "To preserve change ownership by user %s, you must\n" \
1775                       "do \"p4 change -f <change>\" after submitting and\n" \
1776                       "edit the User field."
1777             if pureRenameCopy:
1778                 print "After submitting, renamed files must be re-synced."
1779                 print "Invoke \"p4 sync -f\" on each of these files:"
1780                 for f in pureRenameCopy:
1781                     print "  " + f
1782
1783             print
1784             print "To revert the changes, use \"p4 revert ...\", and delete"
1785             print "the submit template file \"%s\"" % fileName
1786             if filesToAdd:
1787                 print "Since the commit adds new files, they must be deleted:"
1788                 for f in filesToAdd:
1789                     print "  " + f
1790             print
1791             return True
1792
1793         #
1794         # Let the user edit the change description, then submit it.
1795         #
1796         submitted = False
1797
1798         try:
1799             if self.edit_template(fileName):
1800                 # read the edited message and submit
1801                 tmpFile = open(fileName, "rb")
1802                 message = tmpFile.read()
1803                 tmpFile.close()
1804                 if self.isWindows:
1805                     message = message.replace("\r\n", "\n")
1806                 submitTemplate = message[:message.index(separatorLine)]
1807                 p4_write_pipe(['submit', '-i'], submitTemplate)
1808
1809                 if self.preserveUser:
1810                     if p4User:
1811                         # Get last changelist number. Cannot easily get it from
1812                         # the submit command output as the output is
1813                         # unmarshalled.
1814                         changelist = self.lastP4Changelist()
1815                         self.modifyChangelistUser(changelist, p4User)
1816
1817                 # The rename/copy happened by applying a patch that created a
1818                 # new file.  This leaves it writable, which confuses p4.
1819                 for f in pureRenameCopy:
1820                     p4_sync(f, "-f")
1821                 submitted = True
1822
1823         finally:
1824             # skip this patch
1825             if not submitted:
1826                 print "Submission cancelled, undoing p4 changes."
1827                 for f in editedFiles:
1828                     p4_revert(f)
1829                 for f in filesToAdd:
1830                     p4_revert(f)
1831                     os.remove(f)
1832                 for f in filesToDelete:
1833                     p4_revert(f)
1834
1835         os.remove(fileName)
1836         return submitted
1837
1838     # Export git tags as p4 labels. Create a p4 label and then tag
1839     # with that.
1840     def exportGitTags(self, gitTags):
1841         validLabelRegexp = gitConfig("git-p4.labelExportRegexp")
1842         if len(validLabelRegexp) == 0:
1843             validLabelRegexp = defaultLabelRegexp
1844         m = re.compile(validLabelRegexp)
1845
1846         for name in gitTags:
1847
1848             if not m.match(name):
1849                 if verbose:
1850                     print "tag %s does not match regexp %s" % (name, validLabelRegexp)
1851                 continue
1852
1853             # Get the p4 commit this corresponds to
1854             logMessage = extractLogMessageFromGitCommit(name)
1855             values = extractSettingsGitLog(logMessage)
1856
1857             if not values.has_key('change'):
1858                 # a tag pointing to something not sent to p4; ignore
1859                 if verbose:
1860                     print "git tag %s does not give a p4 commit" % name
1861                 continue
1862             else:
1863                 changelist = values['change']
1864
1865             # Get the tag details.
1866             inHeader = True
1867             isAnnotated = False
1868             body = []
1869             for l in read_pipe_lines(["git", "cat-file", "-p", name]):
1870                 l = l.strip()
1871                 if inHeader:
1872                     if re.match(r'tag\s+', l):
1873                         isAnnotated = True
1874                     elif re.match(r'\s*$', l):
1875                         inHeader = False
1876                         continue
1877                 else:
1878                     body.append(l)
1879
1880             if not isAnnotated:
1881                 body = ["lightweight tag imported by git p4\n"]
1882
1883             # Create the label - use the same view as the client spec we are using
1884             clientSpec = getClientSpec()
1885
1886             labelTemplate  = "Label: %s\n" % name
1887             labelTemplate += "Description:\n"
1888             for b in body:
1889                 labelTemplate += "\t" + b + "\n"
1890             labelTemplate += "View:\n"
1891             for depot_side in clientSpec.mappings:
1892                 labelTemplate += "\t%s\n" % depot_side
1893
1894             if self.dry_run:
1895                 print "Would create p4 label %s for tag" % name
1896             elif self.prepare_p4_only:
1897                 print "Not creating p4 label %s for tag due to option" \
1898                       " --prepare-p4-only" % name
1899             else:
1900                 p4_write_pipe(["label", "-i"], labelTemplate)
1901
1902                 # Use the label
1903                 p4_system(["tag", "-l", name] +
1904                           ["%s@%s" % (depot_side, changelist) for depot_side in clientSpec.mappings])
1905
1906                 if verbose:
1907                     print "created p4 label for tag %s" % name
1908
1909     def run(self, args):
1910         if len(args) == 0:
1911             self.master = currentGitBranch()
1912         elif len(args) == 1:
1913             self.master = args[0]
1914             if not branchExists(self.master):
1915                 die("Branch %s does not exist" % self.master)
1916         else:
1917             return False
1918
1919         if self.master:
1920             allowSubmit = gitConfig("git-p4.allowSubmit")
1921             if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1922                 die("%s is not in git-p4.allowSubmit" % self.master)
1923
1924         [upstream, settings] = findUpstreamBranchPoint()
1925         self.depotPath = settings['depot-paths'][0]
1926         if len(self.origin) == 0:
1927             self.origin = upstream
1928
1929         if self.preserveUser:
1930             if not self.canChangeChangelists():
1931                 die("Cannot preserve user names without p4 super-user or admin permissions")
1932
1933         # if not set from the command line, try the config file
1934         if self.conflict_behavior is None:
1935             val = gitConfig("git-p4.conflict")
1936             if val:
1937                 if val not in self.conflict_behavior_choices:
1938                     die("Invalid value '%s' for config git-p4.conflict" % val)
1939             else:
1940                 val = "ask"
1941             self.conflict_behavior = val
1942
1943         if self.verbose:
1944             print "Origin branch is " + self.origin
1945
1946         if len(self.depotPath) == 0:
1947             print "Internal error: cannot locate perforce depot path from existing branches"
1948             sys.exit(128)
1949
1950         self.useClientSpec = False
1951         if gitConfigBool("git-p4.useclientspec"):
1952             self.useClientSpec = True
1953         if self.useClientSpec:
1954             self.clientSpecDirs = getClientSpec()
1955
1956         # Check for the existence of P4 branches
1957         branchesDetected = (len(p4BranchesInGit().keys()) > 1)
1958
1959         if self.useClientSpec and not branchesDetected:
1960             # all files are relative to the client spec
1961             self.clientPath = getClientRoot()
1962         else:
1963             self.clientPath = p4Where(self.depotPath)
1964
1965         if self.clientPath == "":
1966             die("Error: Cannot locate perforce checkout of %s in client view" % self.depotPath)
1967
1968         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1969         self.oldWorkingDirectory = os.getcwd()
1970
1971         # ensure the clientPath exists
1972         new_client_dir = False
1973         if not os.path.exists(self.clientPath):
1974             new_client_dir = True
1975             os.makedirs(self.clientPath)
1976
1977         chdir(self.clientPath, is_client_path=True)
1978         if self.dry_run:
1979             print "Would synchronize p4 checkout in %s" % self.clientPath
1980         else:
1981             print "Synchronizing p4 checkout..."
1982             if new_client_dir:
1983                 # old one was destroyed, and maybe nobody told p4
1984                 p4_sync("...", "-f")
1985             else:
1986                 p4_sync("...")
1987         self.check()
1988
1989         commits = []
1990         if self.master:
1991             commitish = self.master
1992         else:
1993             commitish = 'HEAD'
1994
1995         for line in read_pipe_lines(["git", "rev-list", "--no-merges", "%s..%s" % (self.origin, commitish)]):
1996             commits.append(line.strip())
1997         commits.reverse()
1998
1999         if self.preserveUser or gitConfigBool("git-p4.skipUserNameCheck"):
2000             self.checkAuthorship = False
2001         else:
2002             self.checkAuthorship = True
2003
2004         if self.preserveUser:
2005             self.checkValidP4Users(commits)
2006
2007         #
2008         # Build up a set of options to be passed to diff when
2009         # submitting each commit to p4.
2010         #
2011         if self.detectRenames:
2012             # command-line -M arg
2013             self.diffOpts = "-M"
2014         else:
2015             # If not explicitly set check the config variable
2016             detectRenames = gitConfig("git-p4.detectRenames")
2017
2018             if detectRenames.lower() == "false" or detectRenames == "":
2019                 self.diffOpts = ""
2020             elif detectRenames.lower() == "true":
2021                 self.diffOpts = "-M"
2022             else:
2023                 self.diffOpts = "-M%s" % detectRenames
2024
2025         # no command-line arg for -C or --find-copies-harder, just
2026         # config variables
2027         detectCopies = gitConfig("git-p4.detectCopies")
2028         if detectCopies.lower() == "false" or detectCopies == "":
2029             pass
2030         elif detectCopies.lower() == "true":
2031             self.diffOpts += " -C"
2032         else:
2033             self.diffOpts += " -C%s" % detectCopies
2034
2035         if gitConfigBool("git-p4.detectCopiesHarder"):
2036             self.diffOpts += " --find-copies-harder"
2037
2038         #
2039         # Apply the commits, one at a time.  On failure, ask if should
2040         # continue to try the rest of the patches, or quit.
2041         #
2042         if self.dry_run:
2043             print "Would apply"
2044         applied = []
2045         last = len(commits) - 1
2046         for i, commit in enumerate(commits):
2047             if self.dry_run:
2048                 print " ", read_pipe(["git", "show", "-s",
2049                                       "--format=format:%h %s", commit])
2050                 ok = True
2051             else:
2052                 ok = self.applyCommit(commit)
2053             if ok:
2054                 applied.append(commit)
2055             else:
2056                 if self.prepare_p4_only and i < last:
2057                     print "Processing only the first commit due to option" \
2058                           " --prepare-p4-only"
2059                     break
2060                 if i < last:
2061                     quit = False
2062                     while True:
2063                         # prompt for what to do, or use the option/variable
2064                         if self.conflict_behavior == "ask":
2065                             print "What do you want to do?"
2066                             response = raw_input("[s]kip this commit but apply"
2067                                                  " the rest, or [q]uit? ")
2068                             if not response:
2069                                 continue
2070                         elif self.conflict_behavior == "skip":
2071                             response = "s"
2072                         elif self.conflict_behavior == "quit":
2073                             response = "q"
2074                         else:
2075                             die("Unknown conflict_behavior '%s'" %
2076                                 self.conflict_behavior)
2077
2078                         if response[0] == "s":
2079                             print "Skipping this commit, but applying the rest"
2080                             break
2081                         if response[0] == "q":
2082                             print "Quitting"
2083                             quit = True
2084                             break
2085                     if quit:
2086                         break
2087
2088         chdir(self.oldWorkingDirectory)
2089
2090         if self.dry_run:
2091             pass
2092         elif self.prepare_p4_only:
2093             pass
2094         elif len(commits) == len(applied):
2095             print "All commits applied!"
2096
2097             sync = P4Sync()
2098             if self.branch:
2099                 sync.branch = self.branch
2100             sync.run([])
2101
2102             rebase = P4Rebase()
2103             rebase.rebase()
2104
2105         else:
2106             if len(applied) == 0:
2107                 print "No commits applied."
2108             else:
2109                 print "Applied only the commits marked with '*':"
2110                 for c in commits:
2111                     if c in applied:
2112                         star = "*"
2113                     else:
2114                         star = " "
2115                     print star, read_pipe(["git", "show", "-s",
2116                                            "--format=format:%h %s",  c])
2117                 print "You will have to do 'git p4 sync' and rebase."
2118
2119         if gitConfigBool("git-p4.exportLabels"):
2120             self.exportLabels = True
2121
2122         if self.exportLabels:
2123             p4Labels = getP4Labels(self.depotPath)
2124             gitTags = getGitTags()
2125
2126             missingGitTags = gitTags - p4Labels
2127             self.exportGitTags(missingGitTags)
2128
2129         # exit with error unless everything applied perfectly
2130         if len(commits) != len(applied):
2131                 sys.exit(1)
2132
2133         return True
2134
2135 class View(object):
2136     """Represent a p4 view ("p4 help views"), and map files in a
2137        repo according to the view."""
2138
2139     def __init__(self, client_name):
2140         self.mappings = []
2141         self.client_prefix = "//%s/" % client_name
2142         # cache results of "p4 where" to lookup client file locations
2143         self.client_spec_path_cache = {}
2144
2145     def append(self, view_line):
2146         """Parse a view line, splitting it into depot and client
2147            sides.  Append to self.mappings, preserving order.  This
2148            is only needed for tag creation."""
2149
2150         # Split the view line into exactly two words.  P4 enforces
2151         # structure on these lines that simplifies this quite a bit.
2152         #
2153         # Either or both words may be double-quoted.
2154         # Single quotes do not matter.
2155         # Double-quote marks cannot occur inside the words.
2156         # A + or - prefix is also inside the quotes.
2157         # There are no quotes unless they contain a space.
2158         # The line is already white-space stripped.
2159         # The two words are separated by a single space.
2160         #
2161         if view_line[0] == '"':
2162             # First word is double quoted.  Find its end.
2163             close_quote_index = view_line.find('"', 1)
2164             if close_quote_index <= 0:
2165                 die("No first-word closing quote found: %s" % view_line)
2166             depot_side = view_line[1:close_quote_index]
2167             # skip closing quote and space
2168             rhs_index = close_quote_index + 1 + 1
2169         else:
2170             space_index = view_line.find(" ")
2171             if space_index <= 0:
2172                 die("No word-splitting space found: %s" % view_line)
2173             depot_side = view_line[0:space_index]
2174             rhs_index = space_index + 1
2175
2176         # prefix + means overlay on previous mapping
2177         if depot_side.startswith("+"):
2178             depot_side = depot_side[1:]
2179
2180         # prefix - means exclude this path, leave out of mappings
2181         exclude = False
2182         if depot_side.startswith("-"):
2183             exclude = True
2184             depot_side = depot_side[1:]
2185
2186         if not exclude:
2187             self.mappings.append(depot_side)
2188
2189     def convert_client_path(self, clientFile):
2190         # chop off //client/ part to make it relative
2191         if not clientFile.startswith(self.client_prefix):
2192             die("No prefix '%s' on clientFile '%s'" %
2193                 (self.client_prefix, clientFile))
2194         return clientFile[len(self.client_prefix):]
2195
2196     def update_client_spec_path_cache(self, files):
2197         """ Caching file paths by "p4 where" batch query """
2198
2199         # List depot file paths exclude that already cached
2200         fileArgs = [f['path'] for f in files if f['path'] not in self.client_spec_path_cache]
2201
2202         if len(fileArgs) == 0:
2203             return  # All files in cache
2204
2205         where_result = p4CmdList(["-x", "-", "where"], stdin=fileArgs)
2206         for res in where_result:
2207             if "code" in res and res["code"] == "error":
2208                 # assume error is "... file(s) not in client view"
2209                 continue
2210             if "clientFile" not in res:
2211                 die("No clientFile in 'p4 where' output")
2212             if "unmap" in res:
2213                 # it will list all of them, but only one not unmap-ped
2214                 continue
2215             if gitConfigBool("core.ignorecase"):
2216                 res['depotFile'] = res['depotFile'].lower()
2217             self.client_spec_path_cache[res['depotFile']] = self.convert_client_path(res["clientFile"])
2218
2219         # not found files or unmap files set to ""
2220         for depotFile in fileArgs:
2221             if gitConfigBool("core.ignorecase"):
2222                 depotFile = depotFile.lower()
2223             if depotFile not in self.client_spec_path_cache:
2224                 self.client_spec_path_cache[depotFile] = ""
2225
2226     def map_in_client(self, depot_path):
2227         """Return the relative location in the client where this
2228            depot file should live.  Returns "" if the file should
2229            not be mapped in the client."""
2230
2231         if gitConfigBool("core.ignorecase"):
2232             depot_path = depot_path.lower()
2233
2234         if depot_path in self.client_spec_path_cache:
2235             return self.client_spec_path_cache[depot_path]
2236
2237         die( "Error: %s is not found in client spec path" % depot_path )
2238         return ""
2239
2240 class P4Sync(Command, P4UserMap):
2241     delete_actions = ( "delete", "move/delete", "purge" )
2242
2243     def __init__(self):
2244         Command.__init__(self)
2245         P4UserMap.__init__(self)
2246         self.options = [
2247                 optparse.make_option("--branch", dest="branch"),
2248                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
2249                 optparse.make_option("--changesfile", dest="changesFile"),
2250                 optparse.make_option("--silent", dest="silent", action="store_true"),
2251                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
2252                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
2253                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
2254                                      help="Import into refs/heads/ , not refs/remotes"),
2255                 optparse.make_option("--max-changes", dest="maxChanges",
2256                                      help="Maximum number of changes to import"),
2257                 optparse.make_option("--changes-block-size", dest="changes_block_size", type="int",
2258                                      help="Internal block size to use when iteratively calling p4 changes"),
2259                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
2260                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
2261                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
2262                                      help="Only sync files that are included in the Perforce Client Spec"),
2263                 optparse.make_option("-/", dest="cloneExclude",
2264                                      action="append", type="string",
2265                                      help="exclude depot path"),
2266         ]
2267         self.description = """Imports from Perforce into a git repository.\n
2268     example:
2269     //depot/my/project/ -- to import the current head
2270     //depot/my/project/@all -- to import everything
2271     //depot/my/project/@1,6 -- to import only from revision 1 to 6
2272
2273     (a ... is not needed in the path p4 specification, it's added implicitly)"""
2274
2275         self.usage += " //depot/path[@revRange]"
2276         self.silent = False
2277         self.createdBranches = set()
2278         self.committedChanges = set()
2279         self.branch = ""
2280         self.detectBranches = False
2281         self.detectLabels = False
2282         self.importLabels = False
2283         self.changesFile = ""
2284         self.syncWithOrigin = True
2285         self.importIntoRemotes = True
2286         self.maxChanges = ""
2287         self.changes_block_size = None
2288         self.keepRepoPath = False
2289         self.depotPaths = None
2290         self.p4BranchesInGit = []
2291         self.cloneExclude = []
2292         self.useClientSpec = False
2293         self.useClientSpec_from_options = False
2294         self.clientSpecDirs = None
2295         self.tempBranches = []
2296         self.tempBranchLocation = "refs/git-p4-tmp"
2297         self.largeFileSystem = None
2298
2299         if gitConfig('git-p4.largeFileSystem'):
2300             largeFileSystemConstructor = globals()[gitConfig('git-p4.largeFileSystem')]
2301             self.largeFileSystem = largeFileSystemConstructor(
2302                 lambda git_mode, relPath, contents: self.writeToGitStream(git_mode, relPath, contents)
2303             )
2304
2305         if gitConfig("git-p4.syncFromOrigin") == "false":
2306             self.syncWithOrigin = False
2307
2308     # This is required for the "append" cloneExclude action
2309     def ensure_value(self, attr, value):
2310         if not hasattr(self, attr) or getattr(self, attr) is None:
2311             setattr(self, attr, value)
2312         return getattr(self, attr)
2313
2314     # Force a checkpoint in fast-import and wait for it to finish
2315     def checkpoint(self):
2316         self.gitStream.write("checkpoint\n\n")
2317         self.gitStream.write("progress checkpoint\n\n")
2318         out = self.gitOutput.readline()
2319         if self.verbose:
2320             print "checkpoint finished: " + out
2321
2322     def extractFilesFromCommit(self, commit):
2323         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
2324                              for path in self.cloneExclude]
2325         files = []
2326         fnum = 0
2327         while commit.has_key("depotFile%s" % fnum):
2328             path =  commit["depotFile%s" % fnum]
2329
2330             if [p for p in self.cloneExclude
2331                 if p4PathStartsWith(path, p)]:
2332                 found = False
2333             else:
2334                 found = [p for p in self.depotPaths
2335                          if p4PathStartsWith(path, p)]
2336             if not found:
2337                 fnum = fnum + 1
2338                 continue
2339
2340             file = {}
2341             file["path"] = path
2342             file["rev"] = commit["rev%s" % fnum]
2343             file["action"] = commit["action%s" % fnum]
2344             file["type"] = commit["type%s" % fnum]
2345             files.append(file)
2346             fnum = fnum + 1
2347         return files
2348
2349     def extractJobsFromCommit(self, commit):
2350         jobs = []
2351         jnum = 0
2352         while commit.has_key("job%s" % jnum):
2353             job = commit["job%s" % jnum]
2354             jobs.append(job)
2355             jnum = jnum + 1
2356         return jobs
2357
2358     def stripRepoPath(self, path, prefixes):
2359         """When streaming files, this is called to map a p4 depot path
2360            to where it should go in git.  The prefixes are either
2361            self.depotPaths, or self.branchPrefixes in the case of
2362            branch detection."""
2363
2364         if self.useClientSpec:
2365             # branch detection moves files up a level (the branch name)
2366             # from what client spec interpretation gives
2367             path = self.clientSpecDirs.map_in_client(path)
2368             if self.detectBranches:
2369                 for b in self.knownBranches:
2370                     if path.startswith(b + "/"):
2371                         path = path[len(b)+1:]
2372
2373         elif self.keepRepoPath:
2374             # Preserve everything in relative path name except leading
2375             # //depot/; just look at first prefix as they all should
2376             # be in the same depot.
2377             depot = re.sub("^(//[^/]+/).*", r'\1', prefixes[0])
2378             if p4PathStartsWith(path, depot):
2379                 path = path[len(depot):]
2380
2381         else:
2382             for p in prefixes:
2383                 if p4PathStartsWith(path, p):
2384                     path = path[len(p):]
2385                     break
2386
2387         path = wildcard_decode(path)
2388         return path
2389
2390     def splitFilesIntoBranches(self, commit):
2391         """Look at each depotFile in the commit to figure out to what
2392            branch it belongs."""
2393
2394         if self.clientSpecDirs:
2395             files = self.extractFilesFromCommit(commit)
2396             self.clientSpecDirs.update_client_spec_path_cache(files)
2397
2398         branches = {}
2399         fnum = 0
2400         while commit.has_key("depotFile%s" % fnum):
2401             path =  commit["depotFile%s" % fnum]
2402             found = [p for p in self.depotPaths
2403                      if p4PathStartsWith(path, p)]
2404             if not found:
2405                 fnum = fnum + 1
2406                 continue
2407
2408             file = {}
2409             file["path"] = path
2410             file["rev"] = commit["rev%s" % fnum]
2411             file["action"] = commit["action%s" % fnum]
2412             file["type"] = commit["type%s" % fnum]
2413             fnum = fnum + 1
2414
2415             # start with the full relative path where this file would
2416             # go in a p4 client
2417             if self.useClientSpec:
2418                 relPath = self.clientSpecDirs.map_in_client(path)
2419             else:
2420                 relPath = self.stripRepoPath(path, self.depotPaths)
2421
2422             for branch in self.knownBranches.keys():
2423                 # add a trailing slash so that a commit into qt/4.2foo
2424                 # doesn't end up in qt/4.2, e.g.
2425                 if relPath.startswith(branch + "/"):
2426                     if branch not in branches:
2427                         branches[branch] = []
2428                     branches[branch].append(file)
2429                     break
2430
2431         return branches
2432
2433     def writeToGitStream(self, gitMode, relPath, contents):
2434         self.gitStream.write('M %s inline %s\n' % (gitMode, relPath))
2435         self.gitStream.write('data %d\n' % sum(len(d) for d in contents))
2436         for d in contents:
2437             self.gitStream.write(d)
2438         self.gitStream.write('\n')
2439
2440     # output one file from the P4 stream
2441     # - helper for streamP4Files
2442
2443     def streamOneP4File(self, file, contents):
2444         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
2445         if verbose:
2446             size = int(self.stream_file['fileSize'])
2447             sys.stdout.write('\r%s --> %s (%i MB)\n' % (file['depotFile'], relPath, size/1024/1024))
2448             sys.stdout.flush()
2449
2450         (type_base, type_mods) = split_p4_type(file["type"])
2451
2452         git_mode = "100644"
2453         if "x" in type_mods:
2454             git_mode = "100755"
2455         if type_base == "symlink":
2456             git_mode = "120000"
2457             # p4 print on a symlink sometimes contains "target\n";
2458             # if it does, remove the newline
2459             data = ''.join(contents)
2460             if not data:
2461                 # Some version of p4 allowed creating a symlink that pointed
2462                 # to nothing.  This causes p4 errors when checking out such
2463                 # a change, and errors here too.  Work around it by ignoring
2464                 # the bad symlink; hopefully a future change fixes it.
2465                 print "\nIgnoring empty symlink in %s" % file['depotFile']
2466                 return
2467             elif data[-1] == '\n':
2468                 contents = [data[:-1]]
2469             else:
2470                 contents = [data]
2471
2472         if type_base == "utf16":
2473             # p4 delivers different text in the python output to -G
2474             # than it does when using "print -o", or normal p4 client
2475             # operations.  utf16 is converted to ascii or utf8, perhaps.
2476             # But ascii text saved as -t utf16 is completely mangled.
2477             # Invoke print -o to get the real contents.
2478             #
2479             # On windows, the newlines will always be mangled by print, so put
2480             # them back too.  This is not needed to the cygwin windows version,
2481             # just the native "NT" type.
2482             #
2483             try:
2484                 text = p4_read_pipe(['print', '-q', '-o', '-', '%s@%s' % (file['depotFile'], file['change'])])
2485             except Exception as e:
2486                 if 'Translation of file content failed' in str(e):
2487                     type_base = 'binary'
2488                 else:
2489                     raise e
2490             else:
2491                 if p4_version_string().find('/NT') >= 0:
2492                     text = text.replace('\r\n', '\n')
2493                 contents = [ text ]
2494
2495         if type_base == "apple":
2496             # Apple filetype files will be streamed as a concatenation of
2497             # its appledouble header and the contents.  This is useless
2498             # on both macs and non-macs.  If using "print -q -o xx", it
2499             # will create "xx" with the data, and "%xx" with the header.
2500             # This is also not very useful.
2501             #
2502             # Ideally, someday, this script can learn how to generate
2503             # appledouble files directly and import those to git, but
2504             # non-mac machines can never find a use for apple filetype.
2505             print "\nIgnoring apple filetype file %s" % file['depotFile']
2506             return
2507
2508         # Note that we do not try to de-mangle keywords on utf16 files,
2509         # even though in theory somebody may want that.
2510         pattern = p4_keywords_regexp_for_type(type_base, type_mods)
2511         if pattern:
2512             regexp = re.compile(pattern, re.VERBOSE)
2513             text = ''.join(contents)
2514             text = regexp.sub(r'$\1$', text)
2515             contents = [ text ]
2516
2517         try:
2518             relPath.decode('ascii')
2519         except:
2520             encoding = 'utf8'
2521             if gitConfig('git-p4.pathEncoding'):
2522                 encoding = gitConfig('git-p4.pathEncoding')
2523             relPath = relPath.decode(encoding, 'replace').encode('utf8', 'replace')
2524             if self.verbose:
2525                 print 'Path with non-ASCII characters detected. Used %s to encode: %s ' % (encoding, relPath)
2526
2527         if self.largeFileSystem:
2528             (git_mode, contents) = self.largeFileSystem.processContent(git_mode, relPath, contents)
2529
2530         self.writeToGitStream(git_mode, relPath, contents)
2531
2532     def streamOneP4Deletion(self, file):
2533         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
2534         if verbose:
2535             sys.stdout.write("delete %s\n" % relPath)
2536             sys.stdout.flush()
2537         self.gitStream.write("D %s\n" % relPath)
2538
2539         if self.largeFileSystem and self.largeFileSystem.isLargeFile(relPath):
2540             self.largeFileSystem.removeLargeFile(relPath)
2541
2542     # handle another chunk of streaming data
2543     def streamP4FilesCb(self, marshalled):
2544
2545         # catch p4 errors and complain
2546         err = None
2547         if "code" in marshalled:
2548             if marshalled["code"] == "error":
2549                 if "data" in marshalled:
2550                     err = marshalled["data"].rstrip()
2551
2552         if not err and 'fileSize' in self.stream_file:
2553             required_bytes = int((4 * int(self.stream_file["fileSize"])) - calcDiskFree())
2554             if required_bytes > 0:
2555                 err = 'Not enough space left on %s! Free at least %i MB.' % (
2556                     os.getcwd(), required_bytes/1024/1024
2557                 )
2558
2559         if err:
2560             f = None
2561             if self.stream_have_file_info:
2562                 if "depotFile" in self.stream_file:
2563                     f = self.stream_file["depotFile"]
2564             # force a failure in fast-import, else an empty
2565             # commit will be made
2566             self.gitStream.write("\n")
2567             self.gitStream.write("die-now\n")
2568             self.gitStream.close()
2569             # ignore errors, but make sure it exits first
2570             self.importProcess.wait()
2571             if f:
2572                 die("Error from p4 print for %s: %s" % (f, err))
2573             else:
2574                 die("Error from p4 print: %s" % err)
2575
2576         if marshalled.has_key('depotFile') and self.stream_have_file_info:
2577             # start of a new file - output the old one first
2578             self.streamOneP4File(self.stream_file, self.stream_contents)
2579             self.stream_file = {}
2580             self.stream_contents = []
2581             self.stream_have_file_info = False
2582
2583         # pick up the new file information... for the
2584         # 'data' field we need to append to our array
2585         for k in marshalled.keys():
2586             if k == 'data':
2587                 if 'streamContentSize' not in self.stream_file:
2588                     self.stream_file['streamContentSize'] = 0
2589                 self.stream_file['streamContentSize'] += len(marshalled['data'])
2590                 self.stream_contents.append(marshalled['data'])
2591             else:
2592                 self.stream_file[k] = marshalled[k]
2593
2594         if (verbose and
2595             'streamContentSize' in self.stream_file and
2596             'fileSize' in self.stream_file and
2597             'depotFile' in self.stream_file):
2598             size = int(self.stream_file["fileSize"])
2599             if size > 0:
2600                 progress = 100*self.stream_file['streamContentSize']/size
2601                 sys.stdout.write('\r%s %d%% (%i MB)' % (self.stream_file['depotFile'], progress, int(size/1024/1024)))
2602                 sys.stdout.flush()
2603
2604         self.stream_have_file_info = True
2605
2606     # Stream directly from "p4 files" into "git fast-import"
2607     def streamP4Files(self, files):
2608         filesForCommit = []
2609         filesToRead = []
2610         filesToDelete = []
2611
2612         for f in files:
2613             filesForCommit.append(f)
2614             if f['action'] in self.delete_actions:
2615                 filesToDelete.append(f)
2616             else:
2617                 filesToRead.append(f)
2618
2619         # deleted files...
2620         for f in filesToDelete:
2621             self.streamOneP4Deletion(f)
2622
2623         if len(filesToRead) > 0:
2624             self.stream_file = {}
2625             self.stream_contents = []
2626             self.stream_have_file_info = False
2627
2628             # curry self argument
2629             def streamP4FilesCbSelf(entry):
2630                 self.streamP4FilesCb(entry)
2631
2632             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
2633
2634             p4CmdList(["-x", "-", "print"],
2635                       stdin=fileArgs,
2636                       cb=streamP4FilesCbSelf)
2637
2638             # do the last chunk
2639             if self.stream_file.has_key('depotFile'):
2640                 self.streamOneP4File(self.stream_file, self.stream_contents)
2641
2642     def make_email(self, userid):
2643         if userid in self.users:
2644             return self.users[userid]
2645         else:
2646             return "%s <a@b>" % userid
2647
2648     def streamTag(self, gitStream, labelName, labelDetails, commit, epoch):
2649         """ Stream a p4 tag.
2650         commit is either a git commit, or a fast-import mark, ":<p4commit>"
2651         """
2652
2653         if verbose:
2654             print "writing tag %s for commit %s" % (labelName, commit)
2655         gitStream.write("tag %s\n" % labelName)
2656         gitStream.write("from %s\n" % commit)
2657
2658         if labelDetails.has_key('Owner'):
2659             owner = labelDetails["Owner"]
2660         else:
2661             owner = None
2662
2663         # Try to use the owner of the p4 label, or failing that,
2664         # the current p4 user id.
2665         if owner:
2666             email = self.make_email(owner)
2667         else:
2668             email = self.make_email(self.p4UserId())
2669         tagger = "%s %s %s" % (email, epoch, self.tz)
2670
2671         gitStream.write("tagger %s\n" % tagger)
2672
2673         print "labelDetails=",labelDetails
2674         if labelDetails.has_key('Description'):
2675             description = labelDetails['Description']
2676         else:
2677             description = 'Label from git p4'
2678
2679         gitStream.write("data %d\n" % len(description))
2680         gitStream.write(description)
2681         gitStream.write("\n")
2682
2683     def inClientSpec(self, path):
2684         if not self.clientSpecDirs:
2685             return True
2686         inClientSpec = self.clientSpecDirs.map_in_client(path)
2687         if not inClientSpec and self.verbose:
2688             print('Ignoring file outside of client spec: {0}'.format(path))
2689         return inClientSpec
2690
2691     def hasBranchPrefix(self, path):
2692         if not self.branchPrefixes:
2693             return True
2694         hasPrefix = [p for p in self.branchPrefixes
2695                         if p4PathStartsWith(path, p)]
2696         if not hasPrefix and self.verbose:
2697             print('Ignoring file outside of prefix: {0}'.format(path))
2698         return hasPrefix
2699
2700     def commit(self, details, files, branch, parent = ""):
2701         epoch = details["time"]
2702         author = details["user"]
2703         jobs = self.extractJobsFromCommit(details)
2704
2705         if self.verbose:
2706             print('commit into {0}'.format(branch))
2707
2708         if self.clientSpecDirs:
2709             self.clientSpecDirs.update_client_spec_path_cache(files)
2710
2711         files = [f for f in files
2712             if self.inClientSpec(f['path']) and self.hasBranchPrefix(f['path'])]
2713
2714         if not files and not gitConfigBool('git-p4.keepEmptyCommits'):
2715             print('Ignoring revision {0} as it would produce an empty commit.'
2716                 .format(details['change']))
2717             return
2718
2719         self.gitStream.write("commit %s\n" % branch)
2720         self.gitStream.write("mark :%s\n" % details["change"])
2721         self.committedChanges.add(int(details["change"]))
2722         committer = ""
2723         if author not in self.users:
2724             self.getUserMapFromPerforceServer()
2725         committer = "%s %s %s" % (self.make_email(author), epoch, self.tz)
2726
2727         self.gitStream.write("committer %s\n" % committer)
2728
2729         self.gitStream.write("data <<EOT\n")
2730         self.gitStream.write(details["desc"])
2731         if len(jobs) > 0:
2732             self.gitStream.write("\nJobs: %s" % (' '.join(jobs)))
2733         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s" %
2734                              (','.join(self.branchPrefixes), details["change"]))
2735         if len(details['options']) > 0:
2736             self.gitStream.write(": options = %s" % details['options'])
2737         self.gitStream.write("]\nEOT\n\n")
2738
2739         if len(parent) > 0:
2740             if self.verbose:
2741                 print "parent %s" % parent
2742             self.gitStream.write("from %s\n" % parent)
2743
2744         self.streamP4Files(files)
2745         self.gitStream.write("\n")
2746
2747         change = int(details["change"])
2748
2749         if self.labels.has_key(change):
2750             label = self.labels[change]
2751             labelDetails = label[0]
2752             labelRevisions = label[1]
2753             if self.verbose:
2754                 print "Change %s is labelled %s" % (change, labelDetails)
2755
2756             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
2757                                                 for p in self.branchPrefixes])
2758
2759             if len(files) == len(labelRevisions):
2760
2761                 cleanedFiles = {}
2762                 for info in files:
2763                     if info["action"] in self.delete_actions:
2764                         continue
2765                     cleanedFiles[info["depotFile"]] = info["rev"]
2766
2767                 if cleanedFiles == labelRevisions:
2768                     self.streamTag(self.gitStream, 'tag_%s' % labelDetails['label'], labelDetails, branch, epoch)
2769
2770                 else:
2771                     if not self.silent:
2772                         print ("Tag %s does not match with change %s: files do not match."
2773                                % (labelDetails["label"], change))
2774
2775             else:
2776                 if not self.silent:
2777                     print ("Tag %s does not match with change %s: file count is different."
2778                            % (labelDetails["label"], change))
2779
2780     # Build a dictionary of changelists and labels, for "detect-labels" option.
2781     def getLabels(self):
2782         self.labels = {}
2783
2784         l = p4CmdList(["labels"] + ["%s..." % p for p in self.depotPaths])
2785         if len(l) > 0 and not self.silent:
2786             print "Finding files belonging to labels in %s" % `self.depotPaths`
2787
2788         for output in l:
2789             label = output["label"]
2790             revisions = {}
2791             newestChange = 0
2792             if self.verbose:
2793                 print "Querying files for label %s" % label
2794             for file in p4CmdList(["files"] +
2795                                       ["%s...@%s" % (p, label)
2796                                           for p in self.depotPaths]):
2797                 revisions[file["depotFile"]] = file["rev"]
2798                 change = int(file["change"])
2799                 if change > newestChange:
2800                     newestChange = change
2801
2802             self.labels[newestChange] = [output, revisions]
2803
2804         if self.verbose:
2805             print "Label changes: %s" % self.labels.keys()
2806
2807     # Import p4 labels as git tags. A direct mapping does not
2808     # exist, so assume that if all the files are at the same revision
2809     # then we can use that, or it's something more complicated we should
2810     # just ignore.
2811     def importP4Labels(self, stream, p4Labels):
2812         if verbose:
2813             print "import p4 labels: " + ' '.join(p4Labels)
2814
2815         ignoredP4Labels = gitConfigList("git-p4.ignoredP4Labels")
2816         validLabelRegexp = gitConfig("git-p4.labelImportRegexp")
2817         if len(validLabelRegexp) == 0:
2818             validLabelRegexp = defaultLabelRegexp
2819         m = re.compile(validLabelRegexp)
2820
2821         for name in p4Labels:
2822             commitFound = False
2823
2824             if not m.match(name):
2825                 if verbose:
2826                     print "label %s does not match regexp %s" % (name,validLabelRegexp)
2827                 continue
2828
2829             if name in ignoredP4Labels:
2830                 continue
2831
2832             labelDetails = p4CmdList(['label', "-o", name])[0]
2833
2834             # get the most recent changelist for each file in this label
2835             change = p4Cmd(["changes", "-m", "1"] + ["%s...@%s" % (p, name)
2836                                 for p in self.depotPaths])
2837
2838             if change.has_key('change'):
2839                 # find the corresponding git commit; take the oldest commit
2840                 changelist = int(change['change'])
2841                 if changelist in self.committedChanges:
2842                     gitCommit = ":%d" % changelist       # use a fast-import mark
2843                     commitFound = True
2844                 else:
2845                     gitCommit = read_pipe(["git", "rev-list", "--max-count=1",
2846                         "--reverse", ":/\[git-p4:.*change = %d\]" % changelist], ignore_error=True)
2847                     if len(gitCommit) == 0:
2848                         print "importing label %s: could not find git commit for changelist %d" % (name, changelist)
2849                     else:
2850                         commitFound = True
2851                         gitCommit = gitCommit.strip()
2852
2853                 if commitFound:
2854                     # Convert from p4 time format
2855                     try:
2856                         tmwhen = time.strptime(labelDetails['Update'], "%Y/%m/%d %H:%M:%S")
2857                     except ValueError:
2858                         print "Could not convert label time %s" % labelDetails['Update']
2859                         tmwhen = 1
2860
2861                     when = int(time.mktime(tmwhen))
2862                     self.streamTag(stream, name, labelDetails, gitCommit, when)
2863                     if verbose:
2864                         print "p4 label %s mapped to git commit %s" % (name, gitCommit)
2865             else:
2866                 if verbose:
2867                     print "Label %s has no changelists - possibly deleted?" % name
2868
2869             if not commitFound:
2870                 # We can't import this label; don't try again as it will get very
2871                 # expensive repeatedly fetching all the files for labels that will
2872                 # never be imported. If the label is moved in the future, the
2873                 # ignore will need to be removed manually.
2874                 system(["git", "config", "--add", "git-p4.ignoredP4Labels", name])
2875
2876     def guessProjectName(self):
2877         for p in self.depotPaths:
2878             if p.endswith("/"):
2879                 p = p[:-1]
2880             p = p[p.strip().rfind("/") + 1:]
2881             if not p.endswith("/"):
2882                p += "/"
2883             return p
2884
2885     def getBranchMapping(self):
2886         lostAndFoundBranches = set()
2887
2888         user = gitConfig("git-p4.branchUser")
2889         if len(user) > 0:
2890             command = "branches -u %s" % user
2891         else:
2892             command = "branches"
2893
2894         for info in p4CmdList(command):
2895             details = p4Cmd(["branch", "-o", info["branch"]])
2896             viewIdx = 0
2897             while details.has_key("View%s" % viewIdx):
2898                 paths = details["View%s" % viewIdx].split(" ")
2899                 viewIdx = viewIdx + 1
2900                 # require standard //depot/foo/... //depot/bar/... mapping
2901                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
2902                     continue
2903                 source = paths[0]
2904                 destination = paths[1]
2905                 ## HACK
2906                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
2907                     source = source[len(self.depotPaths[0]):-4]
2908                     destination = destination[len(self.depotPaths[0]):-4]
2909
2910                     if destination in self.knownBranches:
2911                         if not self.silent:
2912                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
2913                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
2914                         continue
2915
2916                     self.knownBranches[destination] = source
2917
2918                     lostAndFoundBranches.discard(destination)
2919
2920                     if source not in self.knownBranches:
2921                         lostAndFoundBranches.add(source)
2922
2923         # Perforce does not strictly require branches to be defined, so we also
2924         # check git config for a branch list.
2925         #
2926         # Example of branch definition in git config file:
2927         # [git-p4]
2928         #   branchList=main:branchA
2929         #   branchList=main:branchB
2930         #   branchList=branchA:branchC
2931         configBranches = gitConfigList("git-p4.branchList")
2932         for branch in configBranches:
2933             if branch:
2934                 (source, destination) = branch.split(":")
2935                 self.knownBranches[destination] = source
2936
2937                 lostAndFoundBranches.discard(destination)
2938
2939                 if source not in self.knownBranches:
2940                     lostAndFoundBranches.add(source)
2941
2942
2943         for branch in lostAndFoundBranches:
2944             self.knownBranches[branch] = branch
2945
2946     def getBranchMappingFromGitBranches(self):
2947         branches = p4BranchesInGit(self.importIntoRemotes)
2948         for branch in branches.keys():
2949             if branch == "master":
2950                 branch = "main"
2951             else:
2952                 branch = branch[len(self.projectName):]
2953             self.knownBranches[branch] = branch
2954
2955     def updateOptionDict(self, d):
2956         option_keys = {}
2957         if self.keepRepoPath:
2958             option_keys['keepRepoPath'] = 1
2959
2960         d["options"] = ' '.join(sorted(option_keys.keys()))
2961
2962     def readOptions(self, d):
2963         self.keepRepoPath = (d.has_key('options')
2964                              and ('keepRepoPath' in d['options']))
2965
2966     def gitRefForBranch(self, branch):
2967         if branch == "main":
2968             return self.refPrefix + "master"
2969
2970         if len(branch) <= 0:
2971             return branch
2972
2973         return self.refPrefix + self.projectName + branch
2974
2975     def gitCommitByP4Change(self, ref, change):
2976         if self.verbose:
2977             print "looking in ref " + ref + " for change %s using bisect..." % change
2978
2979         earliestCommit = ""
2980         latestCommit = parseRevision(ref)
2981
2982         while True:
2983             if self.verbose:
2984                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
2985             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
2986             if len(next) == 0:
2987                 if self.verbose:
2988                     print "argh"
2989                 return ""
2990             log = extractLogMessageFromGitCommit(next)
2991             settings = extractSettingsGitLog(log)
2992             currentChange = int(settings['change'])
2993             if self.verbose:
2994                 print "current change %s" % currentChange
2995
2996             if currentChange == change:
2997                 if self.verbose:
2998                     print "found %s" % next
2999                 return next
3000
3001             if currentChange < change:
3002                 earliestCommit = "^%s" % next
3003             else:
3004                 latestCommit = "%s" % next
3005
3006         return ""
3007
3008     def importNewBranch(self, branch, maxChange):
3009         # make fast-import flush all changes to disk and update the refs using the checkpoint
3010         # command so that we can try to find the branch parent in the git history
3011         self.gitStream.write("checkpoint\n\n");
3012         self.gitStream.flush();
3013         branchPrefix = self.depotPaths[0] + branch + "/"
3014         range = "@1,%s" % maxChange
3015         #print "prefix" + branchPrefix
3016         changes = p4ChangesForPaths([branchPrefix], range, self.changes_block_size)
3017         if len(changes) <= 0:
3018             return False
3019         firstChange = changes[0]
3020         #print "first change in branch: %s" % firstChange
3021         sourceBranch = self.knownBranches[branch]
3022         sourceDepotPath = self.depotPaths[0] + sourceBranch
3023         sourceRef = self.gitRefForBranch(sourceBranch)
3024         #print "source " + sourceBranch
3025
3026         branchParentChange = int(p4Cmd(["changes", "-m", "1", "%s...@1,%s" % (sourceDepotPath, firstChange)])["change"])
3027         #print "branch parent: %s" % branchParentChange
3028         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
3029         if len(gitParent) > 0:
3030             self.initialParents[self.gitRefForBranch(branch)] = gitParent
3031             #print "parent git commit: %s" % gitParent
3032
3033         self.importChanges(changes)
3034         return True
3035
3036     def searchParent(self, parent, branch, target):
3037         parentFound = False
3038         for blob in read_pipe_lines(["git", "rev-list", "--reverse",
3039                                      "--no-merges", parent]):
3040             blob = blob.strip()
3041             if len(read_pipe(["git", "diff-tree", blob, target])) == 0:
3042                 parentFound = True
3043                 if self.verbose:
3044                     print "Found parent of %s in commit %s" % (branch, blob)
3045                 break
3046         if parentFound:
3047             return blob
3048         else:
3049             return None
3050
3051     def importChanges(self, changes):
3052         cnt = 1
3053         for change in changes:
3054             description = p4_describe(change)
3055             self.updateOptionDict(description)
3056
3057             if not self.silent:
3058                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
3059                 sys.stdout.flush()
3060             cnt = cnt + 1
3061
3062             try:
3063                 if self.detectBranches:
3064                     branches = self.splitFilesIntoBranches(description)
3065                     for branch in branches.keys():
3066                         ## HACK  --hwn
3067                         branchPrefix = self.depotPaths[0] + branch + "/"
3068                         self.branchPrefixes = [ branchPrefix ]
3069
3070                         parent = ""
3071
3072                         filesForCommit = branches[branch]
3073
3074                         if self.verbose:
3075                             print "branch is %s" % branch
3076
3077                         self.updatedBranches.add(branch)
3078
3079                         if branch not in self.createdBranches:
3080                             self.createdBranches.add(branch)
3081                             parent = self.knownBranches[branch]
3082                             if parent == branch:
3083                                 parent = ""
3084                             else:
3085                                 fullBranch = self.projectName + branch
3086                                 if fullBranch not in self.p4BranchesInGit:
3087                                     if not self.silent:
3088                                         print("\n    Importing new branch %s" % fullBranch);
3089                                     if self.importNewBranch(branch, change - 1):
3090                                         parent = ""
3091                                         self.p4BranchesInGit.append(fullBranch)
3092                                     if not self.silent:
3093                                         print("\n    Resuming with change %s" % change);
3094
3095                                 if self.verbose:
3096                                     print "parent determined through known branches: %s" % parent
3097
3098                         branch = self.gitRefForBranch(branch)
3099                         parent = self.gitRefForBranch(parent)
3100
3101                         if self.verbose:
3102                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
3103
3104                         if len(parent) == 0 and branch in self.initialParents:
3105                             parent = self.initialParents[branch]
3106                             del self.initialParents[branch]
3107
3108                         blob = None
3109                         if len(parent) > 0:
3110                             tempBranch = "%s/%d" % (self.tempBranchLocation, change)
3111                             if self.verbose:
3112                                 print "Creating temporary branch: " + tempBranch
3113                             self.commit(description, filesForCommit, tempBranch)
3114                             self.tempBranches.append(tempBranch)
3115                             self.checkpoint()
3116                             blob = self.searchParent(parent, branch, tempBranch)
3117                         if blob:
3118                             self.commit(description, filesForCommit, branch, blob)
3119                         else:
3120                             if self.verbose:
3121                                 print "Parent of %s not found. Committing into head of %s" % (branch, parent)
3122                             self.commit(description, filesForCommit, branch, parent)
3123                 else:
3124                     files = self.extractFilesFromCommit(description)
3125                     self.commit(description, files, self.branch,
3126                                 self.initialParent)
3127                     # only needed once, to connect to the previous commit
3128                     self.initialParent = ""
3129             except IOError:
3130                 print self.gitError.read()
3131                 sys.exit(1)
3132
3133     def importHeadRevision(self, revision):
3134         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
3135
3136         details = {}
3137         details["user"] = "git perforce import user"
3138         details["desc"] = ("Initial import of %s from the state at revision %s\n"
3139                            % (' '.join(self.depotPaths), revision))
3140         details["change"] = revision
3141         newestRevision = 0
3142
3143         fileCnt = 0
3144         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
3145
3146         for info in p4CmdList(["files"] + fileArgs):
3147
3148             if 'code' in info and info['code'] == 'error':
3149                 sys.stderr.write("p4 returned an error: %s\n"
3150                                  % info['data'])
3151                 if info['data'].find("must refer to client") >= 0:
3152                     sys.stderr.write("This particular p4 error is misleading.\n")
3153                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
3154                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
3155                 sys.exit(1)
3156             if 'p4ExitCode' in info:
3157                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
3158                 sys.exit(1)
3159
3160
3161             change = int(info["change"])
3162             if change > newestRevision:
3163                 newestRevision = change
3164
3165             if info["action"] in self.delete_actions:
3166                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
3167                 #fileCnt = fileCnt + 1
3168                 continue
3169
3170             for prop in ["depotFile", "rev", "action", "type" ]:
3171                 details["%s%s" % (prop, fileCnt)] = info[prop]
3172
3173             fileCnt = fileCnt + 1
3174
3175         details["change"] = newestRevision
3176
3177         # Use time from top-most change so that all git p4 clones of
3178         # the same p4 repo have the same commit SHA1s.
3179         res = p4_describe(newestRevision)
3180         details["time"] = res["time"]
3181
3182         self.updateOptionDict(details)
3183         try:
3184             self.commit(details, self.extractFilesFromCommit(details), self.branch)
3185         except IOError:
3186             print "IO error with git fast-import. Is your git version recent enough?"
3187             print self.gitError.read()
3188
3189
3190     def run(self, args):
3191         self.depotPaths = []
3192         self.changeRange = ""
3193         self.previousDepotPaths = []
3194         self.hasOrigin = False
3195
3196         # map from branch depot path to parent branch
3197         self.knownBranches = {}
3198         self.initialParents = {}
3199
3200         if self.importIntoRemotes:
3201             self.refPrefix = "refs/remotes/p4/"
3202         else:
3203             self.refPrefix = "refs/heads/p4/"
3204
3205         if self.syncWithOrigin:
3206             self.hasOrigin = originP4BranchesExist()
3207             if self.hasOrigin:
3208                 if not self.silent:
3209                     print 'Syncing with origin first, using "git fetch origin"'
3210                 system("git fetch origin")
3211
3212         branch_arg_given = bool(self.branch)
3213         if len(self.branch) == 0:
3214             self.branch = self.refPrefix + "master"
3215             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
3216                 system("git update-ref %s refs/heads/p4" % self.branch)
3217                 system("git branch -D p4")
3218
3219         # accept either the command-line option, or the configuration variable
3220         if self.useClientSpec:
3221             # will use this after clone to set the variable
3222             self.useClientSpec_from_options = True
3223         else:
3224             if gitConfigBool("git-p4.useclientspec"):
3225                 self.useClientSpec = True
3226         if self.useClientSpec:
3227             self.clientSpecDirs = getClientSpec()
3228
3229         # TODO: should always look at previous commits,
3230         # merge with previous imports, if possible.
3231         if args == []:
3232             if self.hasOrigin:
3233                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
3234
3235             # branches holds mapping from branch name to sha1
3236             branches = p4BranchesInGit(self.importIntoRemotes)
3237
3238             # restrict to just this one, disabling detect-branches
3239             if branch_arg_given:
3240                 short = self.branch.split("/")[-1]
3241                 if short in branches:
3242                     self.p4BranchesInGit = [ short ]
3243             else:
3244                 self.p4BranchesInGit = branches.keys()
3245
3246             if len(self.p4BranchesInGit) > 1:
3247                 if not self.silent:
3248                     print "Importing from/into multiple branches"
3249                 self.detectBranches = True
3250                 for branch in branches.keys():
3251                     self.initialParents[self.refPrefix + branch] = \
3252                         branches[branch]
3253
3254             if self.verbose:
3255                 print "branches: %s" % self.p4BranchesInGit
3256
3257             p4Change = 0
3258             for branch in self.p4BranchesInGit:
3259                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
3260
3261                 settings = extractSettingsGitLog(logMsg)
3262
3263                 self.readOptions(settings)
3264                 if (settings.has_key('depot-paths')
3265                     and settings.has_key ('change')):
3266                     change = int(settings['change']) + 1
3267                     p4Change = max(p4Change, change)
3268
3269                     depotPaths = sorted(settings['depot-paths'])
3270                     if self.previousDepotPaths == []:
3271                         self.previousDepotPaths = depotPaths
3272                     else:
3273                         paths = []
3274                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
3275                             prev_list = prev.split("/")
3276                             cur_list = cur.split("/")
3277                             for i in range(0, min(len(cur_list), len(prev_list))):
3278                                 if cur_list[i] <> prev_list[i]:
3279                                     i = i - 1
3280                                     break
3281
3282                             paths.append ("/".join(cur_list[:i + 1]))
3283
3284                         self.previousDepotPaths = paths
3285
3286             if p4Change > 0:
3287                 self.depotPaths = sorted(self.previousDepotPaths)
3288                 self.changeRange = "@%s,#head" % p4Change
3289                 if not self.silent and not self.detectBranches:
3290                     print "Performing incremental import into %s git branch" % self.branch
3291
3292         # accept multiple ref name abbreviations:
3293         #    refs/foo/bar/branch -> use it exactly
3294         #    p4/branch -> prepend refs/remotes/ or refs/heads/
3295         #    branch -> prepend refs/remotes/p4/ or refs/heads/p4/
3296         if not self.branch.startswith("refs/"):
3297             if self.importIntoRemotes:
3298                 prepend = "refs/remotes/"
3299             else:
3300                 prepend = "refs/heads/"
3301             if not self.branch.startswith("p4/"):
3302                 prepend += "p4/"
3303             self.branch = prepend + self.branch
3304
3305         if len(args) == 0 and self.depotPaths:
3306             if not self.silent:
3307                 print "Depot paths: %s" % ' '.join(self.depotPaths)
3308         else:
3309             if self.depotPaths and self.depotPaths != args:
3310                 print ("previous import used depot path %s and now %s was specified. "
3311                        "This doesn't work!" % (' '.join (self.depotPaths),
3312                                                ' '.join (args)))
3313                 sys.exit(1)
3314
3315             self.depotPaths = sorted(args)
3316
3317         revision = ""
3318         self.users = {}
3319
3320         # Make sure no revision specifiers are used when --changesfile
3321         # is specified.
3322         bad_changesfile = False
3323         if len(self.changesFile) > 0:
3324             for p in self.depotPaths:
3325                 if p.find("@") >= 0 or p.find("#") >= 0:
3326                     bad_changesfile = True
3327                     break
3328         if bad_changesfile:
3329             die("Option --changesfile is incompatible with revision specifiers")
3330
3331         newPaths = []
3332         for p in self.depotPaths:
3333             if p.find("@") != -1:
3334                 atIdx = p.index("@")
3335                 self.changeRange = p[atIdx:]
3336                 if self.changeRange == "@all":
3337                     self.changeRange = ""
3338                 elif ',' not in self.changeRange:
3339                     revision = self.changeRange
3340                     self.changeRange = ""
3341                 p = p[:atIdx]
3342             elif p.find("#") != -1:
3343                 hashIdx = p.index("#")
3344                 revision = p[hashIdx:]
3345                 p = p[:hashIdx]
3346             elif self.previousDepotPaths == []:
3347                 # pay attention to changesfile, if given, else import
3348                 # the entire p4 tree at the head revision
3349                 if len(self.changesFile) == 0:
3350                     revision = "#head"
3351
3352             p = re.sub ("\.\.\.$", "", p)
3353             if not p.endswith("/"):
3354                 p += "/"
3355
3356             newPaths.append(p)
3357
3358         self.depotPaths = newPaths
3359
3360         # --detect-branches may change this for each branch
3361         self.branchPrefixes = self.depotPaths
3362
3363         self.loadUserMapFromCache()
3364         self.labels = {}
3365         if self.detectLabels:
3366             self.getLabels();
3367
3368         if self.detectBranches:
3369             ## FIXME - what's a P4 projectName ?
3370             self.projectName = self.guessProjectName()
3371
3372             if self.hasOrigin:
3373                 self.getBranchMappingFromGitBranches()
3374             else:
3375                 self.getBranchMapping()
3376             if self.verbose:
3377                 print "p4-git branches: %s" % self.p4BranchesInGit
3378                 print "initial parents: %s" % self.initialParents
3379             for b in self.p4BranchesInGit:
3380                 if b != "master":
3381
3382                     ## FIXME
3383                     b = b[len(self.projectName):]
3384                 self.createdBranches.add(b)
3385
3386         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
3387
3388         self.importProcess = subprocess.Popen(["git", "fast-import"],
3389                                               stdin=subprocess.PIPE,
3390                                               stdout=subprocess.PIPE,
3391                                               stderr=subprocess.PIPE);
3392         self.gitOutput = self.importProcess.stdout
3393         self.gitStream = self.importProcess.stdin
3394         self.gitError = self.importProcess.stderr
3395
3396         if revision:
3397             self.importHeadRevision(revision)
3398         else:
3399             changes = []
3400
3401             if len(self.changesFile) > 0:
3402                 output = open(self.changesFile).readlines()
3403                 changeSet = set()
3404                 for line in output:
3405                     changeSet.add(int(line))
3406
3407                 for change in changeSet:
3408                     changes.append(change)
3409
3410                 changes.sort()
3411             else:
3412                 # catch "git p4 sync" with no new branches, in a repo that
3413                 # does not have any existing p4 branches
3414                 if len(args) == 0:
3415                     if not self.p4BranchesInGit:
3416                         die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.")
3417
3418                     # The default branch is master, unless --branch is used to
3419                     # specify something else.  Make sure it exists, or complain
3420                     # nicely about how to use --branch.
3421                     if not self.detectBranches:
3422                         if not branch_exists(self.branch):
3423                             if branch_arg_given:
3424                                 die("Error: branch %s does not exist." % self.branch)
3425                             else:
3426                                 die("Error: no branch %s; perhaps specify one with --branch." %
3427                                     self.branch)
3428
3429                 if self.verbose:
3430                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
3431                                                               self.changeRange)
3432                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange, self.changes_block_size)
3433
3434                 if len(self.maxChanges) > 0:
3435                     changes = changes[:min(int(self.maxChanges), len(changes))]
3436
3437             if len(changes) == 0:
3438                 if not self.silent:
3439                     print "No changes to import!"
3440             else:
3441                 if not self.silent and not self.detectBranches:
3442                     print "Import destination: %s" % self.branch
3443
3444                 self.updatedBranches = set()
3445
3446                 if not self.detectBranches:
3447                     if args:
3448                         # start a new branch
3449                         self.initialParent = ""
3450                     else:
3451                         # build on a previous revision
3452                         self.initialParent = parseRevision(self.branch)
3453
3454                 self.importChanges(changes)
3455
3456                 if not self.silent:
3457                     print ""
3458                     if len(self.updatedBranches) > 0:
3459                         sys.stdout.write("Updated branches: ")
3460                         for b in self.updatedBranches:
3461                             sys.stdout.write("%s " % b)
3462                         sys.stdout.write("\n")
3463
3464         if gitConfigBool("git-p4.importLabels"):
3465             self.importLabels = True
3466
3467         if self.importLabels:
3468             p4Labels = getP4Labels(self.depotPaths)
3469             gitTags = getGitTags()
3470
3471             missingP4Labels = p4Labels - gitTags
3472             self.importP4Labels(self.gitStream, missingP4Labels)
3473
3474         self.gitStream.close()
3475         if self.importProcess.wait() != 0:
3476             die("fast-import failed: %s" % self.gitError.read())
3477         self.gitOutput.close()
3478         self.gitError.close()
3479
3480         # Cleanup temporary branches created during import
3481         if self.tempBranches != []:
3482             for branch in self.tempBranches:
3483                 read_pipe("git update-ref -d %s" % branch)
3484             os.rmdir(os.path.join(os.environ.get("GIT_DIR", ".git"), self.tempBranchLocation))
3485
3486         # Create a symbolic ref p4/HEAD pointing to p4/<branch> to allow
3487         # a convenient shortcut refname "p4".
3488         if self.importIntoRemotes:
3489             head_ref = self.refPrefix + "HEAD"
3490             if not gitBranchExists(head_ref) and gitBranchExists(self.branch):
3491                 system(["git", "symbolic-ref", head_ref, self.branch])
3492
3493         return True
3494
3495 class P4Rebase(Command):
3496     def __init__(self):
3497         Command.__init__(self)
3498         self.options = [
3499                 optparse.make_option("--import-labels", dest="importLabels", action="store_true"),
3500         ]
3501         self.importLabels = False
3502         self.description = ("Fetches the latest revision from perforce and "
3503                             + "rebases the current work (branch) against it")
3504
3505     def run(self, args):
3506         sync = P4Sync()
3507         sync.importLabels = self.importLabels
3508         sync.run([])
3509
3510         return self.rebase()
3511
3512     def rebase(self):
3513         if os.system("git update-index --refresh") != 0:
3514             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
3515         if len(read_pipe("git diff-index HEAD --")) > 0:
3516             die("You have uncommitted changes. Please commit them before rebasing or stash them away with git stash.");
3517
3518         [upstream, settings] = findUpstreamBranchPoint()
3519         if len(upstream) == 0:
3520             die("Cannot find upstream branchpoint for rebase")
3521
3522         # the branchpoint may be p4/foo~3, so strip off the parent
3523         upstream = re.sub("~[0-9]+$", "", upstream)
3524
3525         print "Rebasing the current branch onto %s" % upstream
3526         oldHead = read_pipe("git rev-parse HEAD").strip()
3527         system("git rebase %s" % upstream)
3528         system("git diff-tree --stat --summary -M %s HEAD --" % oldHead)
3529         return True
3530
3531 class P4Clone(P4Sync):
3532     def __init__(self):
3533         P4Sync.__init__(self)
3534         self.description = "Creates a new git repository and imports from Perforce into it"
3535         self.usage = "usage: %prog [options] //depot/path[@revRange]"
3536         self.options += [
3537             optparse.make_option("--destination", dest="cloneDestination",
3538                                  action='store', default=None,
3539                                  help="where to leave result of the clone"),
3540             optparse.make_option("--bare", dest="cloneBare",
3541                                  action="store_true", default=False),
3542         ]
3543         self.cloneDestination = None
3544         self.needsGit = False
3545         self.cloneBare = False
3546
3547     def defaultDestination(self, args):
3548         ## TODO: use common prefix of args?
3549         depotPath = args[0]
3550         depotDir = re.sub("(@[^@]*)$", "", depotPath)
3551         depotDir = re.sub("(#[^#]*)$", "", depotDir)
3552         depotDir = re.sub(r"\.\.\.$", "", depotDir)
3553         depotDir = re.sub(r"/$", "", depotDir)
3554         return os.path.split(depotDir)[1]
3555
3556     def run(self, args):
3557         if len(args) < 1:
3558             return False
3559
3560         if self.keepRepoPath and not self.cloneDestination:
3561             sys.stderr.write("Must specify destination for --keep-path\n")
3562             sys.exit(1)
3563
3564         depotPaths = args
3565
3566         if not self.cloneDestination and len(depotPaths) > 1:
3567             self.cloneDestination = depotPaths[-1]
3568             depotPaths = depotPaths[:-1]
3569
3570         self.cloneExclude = ["/"+p for p in self.cloneExclude]
3571         for p in depotPaths:
3572             if not p.startswith("//"):
3573                 sys.stderr.write('Depot paths must start with "//": %s\n' % p)
3574                 return False
3575
3576         if not self.cloneDestination:
3577             self.cloneDestination = self.defaultDestination(args)
3578
3579         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
3580
3581         if not os.path.exists(self.cloneDestination):
3582             os.makedirs(self.cloneDestination)
3583         chdir(self.cloneDestination)
3584
3585         init_cmd = [ "git", "init" ]
3586         if self.cloneBare:
3587             init_cmd.append("--bare")
3588         retcode = subprocess.call(init_cmd)
3589         if retcode:
3590             raise CalledProcessError(retcode, init_cmd)
3591
3592         if not P4Sync.run(self, depotPaths):
3593             return False
3594
3595         # create a master branch and check out a work tree
3596         if gitBranchExists(self.branch):
3597             system([ "git", "branch", "master", self.branch ])
3598             if not self.cloneBare:
3599                 system([ "git", "checkout", "-f" ])
3600         else:
3601             print 'Not checking out any branch, use ' \
3602                   '"git checkout -q -b master <branch>"'
3603
3604         # auto-set this variable if invoked with --use-client-spec
3605         if self.useClientSpec_from_options:
3606             system("git config --bool git-p4.useclientspec true")
3607
3608         return True
3609
3610 class P4Branches(Command):
3611     def __init__(self):
3612         Command.__init__(self)
3613         self.options = [ ]
3614         self.description = ("Shows the git branches that hold imports and their "
3615                             + "corresponding perforce depot paths")
3616         self.verbose = False
3617
3618     def run(self, args):
3619         if originP4BranchesExist():
3620             createOrUpdateBranchesFromOrigin()
3621
3622         cmdline = "git rev-parse --symbolic "
3623         cmdline += " --remotes"
3624
3625         for line in read_pipe_lines(cmdline):
3626             line = line.strip()
3627
3628             if not line.startswith('p4/') or line == "p4/HEAD":
3629                 continue
3630             branch = line
3631
3632             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
3633             settings = extractSettingsGitLog(log)
3634
3635             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
3636         return True
3637
3638 class HelpFormatter(optparse.IndentedHelpFormatter):
3639     def __init__(self):
3640         optparse.IndentedHelpFormatter.__init__(self)
3641
3642     def format_description(self, description):
3643         if description:
3644             return description + "\n"
3645         else:
3646             return ""
3647
3648 def printUsage(commands):
3649     print "usage: %s <command> [options]" % sys.argv[0]
3650     print ""
3651     print "valid commands: %s" % ", ".join(commands)
3652     print ""
3653     print "Try %s <command> --help for command specific help." % sys.argv[0]
3654     print ""
3655
3656 commands = {
3657     "debug" : P4Debug,
3658     "submit" : P4Submit,
3659     "commit" : P4Submit,
3660     "sync" : P4Sync,
3661     "rebase" : P4Rebase,
3662     "clone" : P4Clone,
3663     "rollback" : P4RollBack,
3664     "branches" : P4Branches
3665 }
3666
3667
3668 def main():
3669     if len(sys.argv[1:]) == 0:
3670         printUsage(commands.keys())
3671         sys.exit(2)
3672
3673     cmdName = sys.argv[1]
3674     try:
3675         klass = commands[cmdName]
3676         cmd = klass()
3677     except KeyError:
3678         print "unknown command %s" % cmdName
3679         print ""
3680         printUsage(commands.keys())
3681         sys.exit(2)
3682
3683     options = cmd.options
3684     cmd.gitdir = os.environ.get("GIT_DIR", None)
3685
3686     args = sys.argv[2:]
3687
3688     options.append(optparse.make_option("--verbose", "-v", dest="verbose", action="store_true"))
3689     if cmd.needsGit:
3690         options.append(optparse.make_option("--git-dir", dest="gitdir"))
3691
3692     parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
3693                                    options,
3694                                    description = cmd.description,
3695                                    formatter = HelpFormatter())
3696
3697     (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
3698     global verbose
3699     verbose = cmd.verbose
3700     if cmd.needsGit:
3701         if cmd.gitdir == None:
3702             cmd.gitdir = os.path.abspath(".git")
3703             if not isValidGitDir(cmd.gitdir):
3704                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
3705                 if os.path.exists(cmd.gitdir):
3706                     cdup = read_pipe("git rev-parse --show-cdup").strip()
3707                     if len(cdup) > 0:
3708                         chdir(cdup);
3709
3710         if not isValidGitDir(cmd.gitdir):
3711             if isValidGitDir(cmd.gitdir + "/.git"):
3712                 cmd.gitdir += "/.git"
3713             else:
3714                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
3715
3716         os.environ["GIT_DIR"] = cmd.gitdir
3717
3718     if not cmd.run(args):
3719         parser.print_help()
3720         sys.exit(2)
3721
3722
3723 if __name__ == '__main__':
3724     main()