# found in the LICENSE file.
+import cStringIO
import os
import re
import subprocess
VERBOSE = False
# The longest any single subprocess will be allowed to run.
-TIMEOUT = 20 * 60
+TIMEOUT = 40 * 60
+
+class AbnormalExit(Exception):
+ pass
-def GetStatusOutput(cmd, cwd=None):
+class StdioBuffer(object):
+ def __init__(self, name, out_queue):
+ self.closed = False
+ self.line_buffer = cStringIO.StringIO()
+ self.name = name
+ self.out_q = out_queue
+
+ def write(self, msg):
+ """Write into the buffer. Only one thread should call write() at a time."""
+ assert not self.closed
+ self.line_buffer.write(msg)
+ # We can use '\n' instead of os.linesep because universal newlines is
+ # set to true below.
+ if '\n' in msg:
+ # We can assert that lines is at least 2 items if '\n' is present.
+ lines = self.line_buffer.getvalue().split('\n')
+ for line in lines[:-1]:
+ self.out_q.put('%s> %s' % (self.name, line))
+ self.line_buffer.close()
+ self.line_buffer = cStringIO.StringIO()
+ self.line_buffer.write(lines[-1])
+
+ def close(self):
+ # Empty out the line buffer.
+ self.write('\n')
+ self.out_q.put(None)
+ self.closed = True
+
+
+def GetStatusOutput(cmd, cwd=None, out_buffer=None):
"""Return (status, output) of executing cmd in a shell."""
if VERBOSE:
print ''
thr.stdout = ''
thr.stderr = '<timeout>'
try:
- proc = subprocess.Popen(cmd, shell=True, universal_newlines=True, cwd=cwd,
- stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
- (stdout, _) = proc.communicate()
+ if out_buffer:
+ proc = subprocess.Popen(cmd, shell=True,
+ cwd=cwd, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ while True:
+ buf = proc.stdout.read(1)
+ if buf == '\r': # We want carriage returns in Linux to be newlines.
+ buf = '\n'
+ if not buf:
+ break
+ out_buffer.write(buf)
+ stdout = ''
+ proc.wait()
+ out_buffer.close()
+ else:
+ proc = subprocess.Popen(cmd, shell=True, universal_newlines=True,
+ cwd=cwd, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ (stdout, _) = proc.communicate()
except Exception, e:
thr.status = -1
thr.stdout = ''
return (thr.status, thr.stdout)
-def Git(git_repo, command, is_mirror=False):
+def Git(git_repo, command, is_mirror=False, out_buffer=None):
"""Execute a git command within a local git repo."""
if is_mirror:
- cmd = 'git --git-dir=%s %s' % (git_repo, command)
+ if git_repo:
+ cmd = 'git --git-dir=%s %s' % (git_repo, command)
+ else:
+ cmd = 'git %s' % command
cwd = None
else:
cmd = 'git %s' % command
cwd = git_repo
- (status, output) = GetStatusOutput(cmd, cwd)
- if status != 0:
+ (status, output) = GetStatusOutput(cmd, cwd, out_buffer)
+ # For Abnormal Exit, Windows returns -1, Posix returns 128.
+ if status in [-1, 128]:
+ raise AbnormalExit('Failed to run %s. Exited Abnormally. output %s' %
+ (cmd, output))
+ elif status != 0:
raise Exception('Failed to run %s. error %d. output %s' % (cmd, status,
output))
return (status, output)
-def Clone(git_url, git_repo, is_mirror):
+def GetCacheRepoDir(git_url, cache_dir):
+ """Assuming we used git_cache to populate a cache, get the repo directory."""
+ _, out = Git(None, 'cache exists --cache-dir=%s %s' % (cache_dir, git_url),
+ is_mirror=True)
+ return out.strip()
+
+
+def Clone(git_url, git_repo, is_mirror, out_queue=None, cache_dir=None,
+ shallow=False):
"""Clone a repository."""
- cmd = 'clone%s %s %s' % (' --mirror' if is_mirror else '', git_url, git_repo)
+ repo_name = git_url.split('/')[-1]
+ if out_queue:
+ buf = StdioBuffer(repo_name, out_queue)
+ else:
+ buf = None
+
+ if is_mirror == 'bare':
+ if shallow:
+ if 'adobe' in git_url:
+ # --shallow by default checks out 10000 revision, but for really large
+ # repos like adobe ones, we want significantly less than 10000.
+ shallow_arg = '--depth 10 '
+ else:
+ shallow_arg = '--shallow '
+ else:
+ shallow_arg = ''
+ cmd = 'cache populate -v --cache-dir %s %s%s' % (cache_dir, shallow_arg,
+ git_url)
+ return Git(None, cmd, is_mirror=True, out_buffer=buf)
+
+ cmd = 'clone'
+ if is_mirror:
+ cmd += ' --mirror'
+ cmd += ' %s %s' % (git_url, git_repo)
+
if not is_mirror and not os.path.exists(git_repo):
os.makedirs(git_repo)
- return Git(git_repo, cmd, is_mirror)
+
+ return Git(None, cmd, is_mirror, buf)
def Fetch(git_repo, git_url, is_mirror):
def _FindRevForCommitish(git_repo, commitish, is_mirror):
_, output = Git(git_repo, 'cat-file commit %s' % commitish, is_mirror)
match = re.match(r'git-svn-id: [^\s@]+@(\d+) \S+$', output.splitlines()[-1])
- assert match, 'no match on %s' % output
- return int(match.group(1))
+ if match:
+ return int(match.group(1))
+ else:
+ # The last commit isn't from svn, but maybe the repo was converted to pure
+ # git at some point, so the last svn commit is somewhere farther back.
+ _, output = Git(
+ git_repo, ('log -E --grep="^git-svn-id: [^@]*@[0-9]* [A-Za-z0-9-]*$" '
+ '-1 --format="%%H" %s') % commitish, is_mirror)
+ assert output, 'no match on %s' % commitish
# Check if svn_rev is newer than the current refspec revision.
- found_rev = _FindRevForCommitish(git_repo, refspec, is_mirror)
- if found_rev < int(svn_rev) and fetch_url:
+ try:
+ found_rev = _FindRevForCommitish(git_repo, refspec, is_mirror)
+ # Sometimes this fails because it's looking in a branch that hasn't been
+ # fetched from upstream yet. Let it fetch and try again.
+ except AbnormalExit:
+ found_rev = None
+ if (not found_rev or found_rev < int(svn_rev)) and fetch_url:
if VERBOSE:
print 'Fetching %s %s [%s < %s]' % (git_repo, refspec, found_rev, svn_rev)
Fetch(git_repo, fetch_url, is_mirror)
+ found_rev = _FindRevForCommitish(git_repo, refspec, is_mirror)
# Find the first commit matching the given git-svn-id regex.
_, output = Git(
is_mirror)
output = output.strip()
if not re.match('^[0-9a-fA-F]{40}$', output):
- raise SearchError('Cannot find revision %s in %s' % (svn_rev, git_repo))
+ raise SearchError('Cannot find revision %s in %s:%s' % (svn_rev, git_repo,
+ refspec))
# Check if it actually matched the svn_rev that was requested.
found_rev = _FindRevForCommitish(git_repo, output, is_mirror)
projects / platform / framework / web / crosswalk.git / blobdiff