1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Common functions for interacting with git and repo."""
12 # pylint: disable=W0402
18 # TODO(build): Fix this.
19 # This should be absolute import, but that requires fixing all
20 # relative imports first.
21 _path = os.path.realpath(__file__)
22 _path = os.path.normpath(os.path.join(os.path.dirname(_path), '..', '..'))
23 sys.path.insert(0, _path)
24 from chromite.cbuildbot import constants
25 from chromite.lib import cros_build_lib
26 from chromite.lib import osutils
27 from chromite.lib import retry_util
28 # Now restore it so that relative scripts don't get cranky.
32 # Retry a git operation if git returns a error response with any of these
33 # messages. It's all observed 'bad' GoB responses so far.
34 GIT_TRANSIENT_ERRORS = (
36 r'! \[remote rejected\].*\(error in hook\)',
39 r'! \[remote rejected\].*\(failed to lock\)',
42 r'! \[remote rejected\].*\(error in Gerrit backend\)',
45 r'remote error: Internal Server Error',
48 r'fatal: Couldn\'t find remote ref ',
51 r'git fetch_pack: expected ACK/NAK, got',
54 r'protocol error: bad pack header',
57 r'The remote end hung up unexpectedly',
60 r'TLS packet with unexpected length was received',
63 r'RPC failed; result=\d+, HTTP code = \d+',
66 r'The requested URL returned error: 500 while accessing',
69 r'Connection timed out',
72 GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS),
75 DEFAULT_RETRY_INTERVAL = 3
79 class RemoteRef(object):
80 """Object representing a remote ref.
82 A remote ref encapsulates both a remote (e.g., 'origin',
83 'https://chromium.googlesource.com/chromiumos/chromite.git', etc.) and a ref
84 name (e.g., 'refs/heads/master').
87 def __init__(self, remote, ref):
92 def FindRepoDir(path):
93 """Returns the nearest higher-level repo dir from the specified path.
96 path: The path to use. Defaults to cwd.
98 return osutils.FindInPathParents(
99 '.repo', path, test_func=os.path.isdir)
102 def FindRepoCheckoutRoot(path):
103 """Get the root of your repo managed checkout."""
104 repo_dir = FindRepoDir(path)
106 return os.path.dirname(repo_dir)
111 def IsSubmoduleCheckoutRoot(path, remote, url):
112 """Tests to see if a directory is the root of a git submodule checkout.
115 path: The directory to test.
116 remote: The remote to compare the |url| with.
117 url: The exact URL the |remote| needs to be pointed at.
119 if os.path.isdir(path):
120 remote_url = cros_build_lib.RunCommand(
121 ['git', '--git-dir', path, 'config', 'remote.%s.url' % remote],
122 redirect_stdout=True, debug_level=logging.DEBUG,
123 error_code_ok=True).output.strip()
124 if remote_url == url:
129 def ReinterpretPathForChroot(path):
130 """Returns reinterpreted path from outside the chroot for use inside.
133 path: The path to reinterpret. Must be in src tree.
135 root_path = os.path.join(FindRepoDir(path), '..')
137 path_abs_path = os.path.abspath(path)
138 root_abs_path = os.path.abspath(root_path)
140 # Strip the repository root from the path and strip first /.
141 relative_path = path_abs_path.replace(root_abs_path, '')[1:]
143 if relative_path == path_abs_path:
144 raise Exception('Error: path is outside your src tree, cannot reinterpret.')
146 new_path = os.path.join('/home', os.getenv('USER'), 'trunk', relative_path)
151 """Checks if there's a git repo rooted at a directory."""
152 return os.path.isdir(os.path.join(cwd, '.git'))
155 def IsGitRepositoryCorrupted(cwd):
156 """Verify that the specified git repository is not corrupted.
159 cwd: The git repository to verify.
162 True if the repository is corrupted.
164 cmd = ['fsck', '--no-progress', '--no-dangling']
168 except cros_build_lib.RunCommandError as ex:
169 logging.warn(str(ex))
173 _HEX_CHARS = frozenset(string.hexdigits)
176 def IsSHA1(value, full=True):
177 """Returns True if the given value looks like a sha1.
179 If full is True, then it must be full length- 40 chars. If False, >=6, and
182 if not all(x in _HEX_CHARS for x in value):
187 return l >= 6 and l <= 40
190 def IsRefsTags(value):
191 """Return True if the given value looks like a tag.
193 Currently this is identified via refs/tags/ prefixing.
195 return value.startswith('refs/tags/')
198 def GetGitRepoRevision(cwd, branch='HEAD'):
199 """Find the revision of a branch.
201 Defaults to current branch.
203 return RunGit(cwd, ['rev-parse', branch]).output.strip()
206 def DoesCommitExistInRepo(cwd, commit):
207 """Determine whether a commit (SHA1 or ref) exists in a repo.
210 cwd: A directory within the project repo.
211 commit: The commit to look for. This can be a SHA1 or it can be a ref.
214 True if the commit exists in the repo.
217 RunGit(cwd, ['rev-list', '-n1', commit, '--'])
218 except cros_build_lib.RunCommandError as e:
219 if e.result.returncode == 128:
225 def GetCurrentBranch(cwd):
226 """Returns current branch of a repo, and None if repo is on detached HEAD."""
228 ret = RunGit(cwd, ['symbolic-ref', '-q', 'HEAD'])
229 return StripRefsHeads(ret.output.strip(), False)
230 except cros_build_lib.RunCommandError as e:
231 if e.result.returncode != 1:
236 def StripRefsHeads(ref, strict=True):
237 """Remove leading 'refs/heads/' from a ref name.
239 If strict is True, an Exception is thrown if the ref doesn't start with
240 refs/heads. If strict is False, the original ref is returned.
242 if not ref.startswith('refs/heads/') and strict:
243 raise Exception('Ref name %s does not start with refs/heads/' % ref)
245 return ref.replace('refs/heads/', '')
249 """Remove leading 'refs/heads', 'refs/remotes/[^/]+/' from a ref name."""
250 ref = StripRefsHeads(ref, False)
251 if ref.startswith('refs/remotes/'):
252 return ref.split('/', 3)[-1]
256 def NormalizeRef(ref):
257 """Convert git branch refs into fully qualified form."""
258 if ref and not ref.startswith('refs/'):
259 ref = 'refs/heads/%s' % ref
263 def NormalizeRemoteRef(remote, ref):
264 """Convert git branch refs into fully qualified remote form."""
266 # Support changing local ref to remote ref, or changing the remote
270 if not ref.startswith('refs/'):
271 ref = 'refs/remotes/%s/%s' % (remote, ref)
276 class ProjectCheckout(dict):
277 """Attributes of a given project in the manifest checkout.
279 TODO(davidjames): Convert this into an ordinary object instead of a dict.
282 def __init__(self, attrs):
286 attrs: The attributes associated with this checkout, as a dictionary.
288 dict.__init__(self, attrs)
290 def AssertPushable(self):
291 """Verify that it is safe to push changes to this repository."""
292 if not self['pushable']:
293 remote = self['remote']
294 raise AssertionError('Remote %s is not pushable.' % (remote,))
296 def IsBranchableProject(self):
297 """Return whether this project is hosted on ChromeOS git servers."""
299 self['remote'] in constants.CROS_REMOTES and
300 re.match(constants.BRANCHABLE_PROJECTS[self['remote']], self['name']))
302 def IsPatchable(self):
303 """Returns whether this project is patchable.
305 For projects that get checked out at multiple paths and/or branches,
306 this method can be used to determine which project path a patch
307 should be applied to.
310 True if the project corresponding to the checkout is patchable.
312 # There are 2 ways we determine if a project is patchable.
313 # - For an unversioned manifest, if the 'revision' is a raw SHA1 hash
314 # and not a named branch, assume it is a pinned project path and can not
316 # - For a versioned manifest (generated via repo -r), repo will set
317 # revision to the actual git sha1 ref, and add an 'upstream'
318 # field corresponding to branch name in the original manifest. For
319 # a project with a SHA1 'revision' but no named branch in the
320 # 'upstream' field, assume it can not be patched.
321 return not IsSHA1(self.get('upstream', self['revision']))
323 def GetPath(self, absolute=False):
324 """Get the path to the checkout.
327 absolute: If True, return an absolute path. If False,
328 return a path relative to the repo root.
330 return self['local_path'] if absolute else self['path']
333 class Manifest(object):
334 """SAX handler that parses the manifest document.
337 checkouts_by_name: A dictionary mapping the names for <project> tags to a
338 list of ProjectCheckout objects.
339 checkouts_by_path: A dictionary mapping paths for <project> tags to a single
340 ProjectCheckout object.
341 default: The attributes of the <default> tag.
342 includes: A list of XML files that should be pulled in to the manifest.
343 These includes are represented as a list of (name, path) tuples.
344 manifest_include_dir: If given, this is where to start looking for
346 projects: DEPRECATED. A dictionary mapping the names for <project> tags to
347 a single ProjectCheckout object. This is now deprecated, since each
348 project can map to multiple ProjectCheckout objects.
349 remotes: A dictionary mapping <remote> tags to the associated attributes.
350 revision: The revision of the manifest repository. If not specified, this
356 def __init__(self, source, manifest_include_dir=None):
357 """Initialize this instance.
360 source: The path to the manifest to parse. May be a file handle.
361 manifest_include_dir: If given, this is where to start looking for
366 self.checkouts_by_path = {}
367 self.checkouts_by_name = {}
371 self.manifest_include_dir = manifest_include_dir
372 self._RunParser(source)
373 self.includes = tuple(self.includes)
375 def _RunParser(self, source, finalize=True):
376 parser = sax.make_parser()
377 handler = sax.handler.ContentHandler()
378 handler.startElement = self._ProcessElement
379 parser.setContentHandler(handler)
382 self._FinalizeAllProjectData()
384 def _ProcessElement(self, name, attrs):
385 """Stores the default manifest properties and per-project overrides."""
386 attrs = dict(attrs.items())
387 if name == 'default':
389 elif name == 'remote':
390 attrs.setdefault('alias', attrs['name'])
391 self.remotes[attrs['name']] = attrs
392 elif name == 'project':
393 self.checkouts_by_path[attrs.get('path', attrs['name'])] = attrs
394 self.checkouts_by_name.setdefault(attrs['name'], []).append(attrs)
395 elif name == 'manifest':
396 self.revision = attrs.get('revision')
397 elif name == 'include':
398 if self.manifest_include_dir is None:
400 errno.ENOENT, 'No manifest_include_dir given, but an include was '
401 'encountered; attrs=%r' % (attrs,))
402 # Include is calculated relative to the manifest that has the include;
403 # thus set the path temporarily to the dirname of the target.
404 original_include_dir = self.manifest_include_dir
405 include_path = os.path.realpath(
406 os.path.join(original_include_dir, attrs['name']))
407 self.includes.append((attrs['name'], include_path))
408 self._RunParser(include_path, finalize=False)
410 def _FinalizeAllProjectData(self):
411 """Rewrite projects mixing defaults in and adding our attributes."""
412 for path_data in self.checkouts_by_path.itervalues():
413 self._FinalizeProjectData(path_data)
415 def _FinalizeProjectData(self, attrs):
416 """Sets up useful properties for a project.
419 attrs: The attribute dictionary of a <project> tag.
421 for key in ('remote', 'revision'):
422 attrs.setdefault(key, self.default.get(key))
424 remote = attrs['remote']
425 assert remote in self.remotes
426 remote_name = attrs['remote_alias'] = self.remotes[remote]['alias']
428 # 'repo manifest -r' adds an 'upstream' attribute to the project tag for the
429 # manifests it generates. We can use the attribute to get a valid branch
430 # instead of a sha1 for these types of manifests.
431 upstream = attrs.get('upstream', attrs['revision'])
433 # The current version of repo we use has a bug: When you create a new
434 # repo checkout from a revlocked manifest, the 'upstream' attribute will
435 # just point at a SHA1. The default revision will still be correct,
436 # however. For now, return the default revision as our best guess as to
437 # what the upstream branch for this repository would be. This guess may
438 # sometimes be wrong, but it's correct for all of the repositories where
439 # we need to push changes (e.g., the overlays).
440 # TODO(davidjames): Either fix the repo bug, or update our logic here to
441 # check the manifest repository to find the right tracking branch.
442 upstream = self.default.get('revision', 'refs/heads/master')
444 attrs['tracking_branch'] = 'refs/remotes/%s/%s' % (
445 remote_name, StripRefs(upstream),
448 attrs['pushable'] = remote in constants.GIT_REMOTES
449 if attrs['pushable']:
450 attrs['push_remote'] = remote
451 attrs['push_remote_url'] = constants.GIT_REMOTES[remote]
452 attrs['push_url'] = '%s/%s' % (attrs['push_remote_url'], attrs['name'])
453 groups = set(attrs.get('groups', 'default').replace(',', ' ').split())
454 groups.add('default')
455 attrs['groups'] = frozenset(groups)
457 # Compute the local ref space.
458 # Sanitize a couple path fragments to simplify assumptions in this
459 # class, and in consuming code.
460 attrs.setdefault('path', attrs['name'])
461 for key in ('name', 'path'):
462 attrs[key] = os.path.normpath(attrs[key])
465 def _GetManifestHash(source, ignore_missing=False):
466 if isinstance(source, basestring):
468 # TODO(build): convert this to osutils.ReadFile once these
469 # classes are moved out into their own module (if possible;
470 # may still be cyclic).
471 with open(source, 'rb') as f:
472 return hashlib.md5(f.read()).hexdigest()
473 except EnvironmentError as e:
474 if e.errno != errno.ENOENT or not ignore_missing:
477 md5 = hashlib.md5(source.read()).hexdigest()
482 def Cached(cls, source, manifest_include_dir=None):
483 """Return an instance, reusing an existing one if possible.
485 May be a seekable filehandle, or a filepath.
486 See __init__ for an explanation of these arguments.
489 md5 = cls._GetManifestHash(source)
490 obj, sources = cls._instance_cache.get(md5, (None, ()))
491 if manifest_include_dir is None and sources:
492 # We're being invoked in a different way than the orignal
493 # caching; disregard the cached entry.
494 # Most likely, the instantiation will explode; let it fly.
495 obj, sources = None, ()
496 for include_target, target_md5 in sources:
497 if cls._GetManifestHash(include_target, True) != target_md5:
501 obj = cls(source, manifest_include_dir=manifest_include_dir)
502 sources = tuple((abspath, cls._GetManifestHash(abspath))
503 for (target, abspath) in obj.includes)
504 cls._instance_cache[md5] = (obj, sources)
509 class ManifestCheckout(Manifest):
510 """A Manifest Handler for a specific manifest checkout."""
514 # pylint: disable=W0221
515 def __init__(self, path, manifest_path=None, search=True):
516 """Initialize this instance.
519 path: Path into a manifest checkout (doesn't have to be the root).
520 manifest_path: If supplied, the manifest to use. Else the manifest
521 in the root of the checkout is used. May be a seekable file handle.
522 search: If True, the path can point into the repo, and the root will
523 be found automatically. If False, the path *must* be the root, else
524 an OSError ENOENT will be thrown.
527 OSError: if a failure occurs.
529 self.root, manifest_path = self._NormalizeArgs(
530 path, manifest_path, search=search)
532 self.manifest_path = os.path.realpath(manifest_path)
533 manifest_include_dir = os.path.dirname(self.manifest_path)
534 self.manifest_branch = self._GetManifestsBranch(self.root)
535 self._content_merging = {}
536 Manifest.__init__(self, self.manifest_path,
537 manifest_include_dir=manifest_include_dir)
540 def _NormalizeArgs(path, manifest_path=None, search=True):
541 root = FindRepoCheckoutRoot(path)
543 raise OSError(errno.ENOENT, "Couldn't find repo root: %s" % (path,))
544 root = os.path.normpath(os.path.realpath(root))
546 if os.path.normpath(os.path.realpath(path)) != root:
547 raise OSError(errno.ENOENT, 'Path %s is not a repo root, and search '
548 'is disabled.' % path)
549 if manifest_path is None:
550 manifest_path = os.path.join(root, '.repo', 'manifest.xml')
551 return root, manifest_path
553 def FindCheckouts(self, project, branch=None, only_patchable=False):
554 """Returns the list of checkouts for a given |project|/|branch|.
557 project: Project name to search for.
558 branch: Branch to use.
559 only_patchable: Restrict search to patchable project paths.
562 A list of ProjectCheckout objects.
565 for checkout in self.checkouts_by_name.get(project, []):
566 if project == checkout['name']:
567 if only_patchable and not checkout.IsPatchable():
569 tracking_branch = checkout['tracking_branch']
570 if branch is None or StripRefs(branch) == StripRefs(tracking_branch):
571 checkouts.append(checkout)
574 def FindCheckout(self, project, branch=None, strict=True):
575 """Returns the checkout associated with a given project/branch.
578 project: The project to look for.
579 branch: The branch that the project is tracking.
580 strict: Raise AssertionError if a checkout cannot be found.
583 A ProjectCheckout object.
586 AssertionError if there is more than one checkout associated with the
587 given project/branch combination.
589 checkouts = self.FindCheckouts(project, branch)
590 if len(checkouts) < 1:
592 raise AssertionError('Could not find checkout of %s' % (project,))
594 elif len(checkouts) > 1:
595 raise AssertionError('Too many checkouts found for %s' % project)
598 def ListCheckouts(self):
599 """List the checkouts in the manifest.
602 A list of ProjectCheckout objects.
604 return self.checkouts_by_path.values()
606 def FindCheckoutFromPath(self, path, strict=True):
607 """Find the associated checkouts for a given |path|.
609 The |path| can either be to the root of a project, or within the
610 project itself (chromite.cbuildbot for example). It may be relative
611 to the repo root, or an absolute path. If |path| is not within a
612 checkout, return None.
615 path: Path to examine.
616 strict: If True, fail when no checkout is found.
619 None if no checkout is found, else the checkout.
621 # Realpath everything sans the target to keep people happy about
622 # how symlinks are handled; exempt the final node since following
623 # through that is unlikely even remotely desired.
624 tmp = os.path.join(self.root, os.path.dirname(path))
625 path = os.path.join(os.path.realpath(tmp), os.path.basename(path))
626 path = os.path.normpath(path) + '/'
628 for checkout in self.ListCheckouts():
629 if path.startswith(checkout['local_path'] + '/'):
630 candidates.append((checkout['path'], checkout))
634 raise AssertionError('Could not find repo project at %s' % (path,))
637 # The checkout with the greatest common path prefix is the owner of
638 # the given pathway. Return that.
639 return max(candidates)[1]
641 def _FinalizeAllProjectData(self):
642 """Rewrite projects mixing defaults in and adding our attributes."""
643 Manifest._FinalizeAllProjectData(self)
644 for key, value in self.checkouts_by_path.iteritems():
645 self.checkouts_by_path[key] = ProjectCheckout(value)
646 for key, value in self.checkouts_by_name.iteritems():
647 self.checkouts_by_name[key] = \
648 [ProjectCheckout(x) for x in value]
650 def _FinalizeProjectData(self, attrs):
651 Manifest._FinalizeProjectData(self, attrs)
652 attrs['local_path'] = os.path.join(self.root, attrs['path'])
655 def _GetManifestsBranch(root):
656 """Get the tracking branch of the manifest repository.
661 # Suppress the normal "if it ain't refs/heads, we don't want none o' that"
662 # check for the merge target; repo writes the ambigious form of the branch
663 # target for `repo init -u url -b some-branch` usages (aka, 'master'
664 # instead of 'refs/heads/master').
665 path = os.path.join(root, '.repo', 'manifests')
666 current_branch = GetCurrentBranch(path)
667 if current_branch != 'default':
668 raise OSError(errno.ENOENT,
669 'Manifest repository at %s is checked out to %s. '
670 "It should be checked out to 'default'."
671 % (root, 'detached HEAD' if current_branch is None
672 else current_branch))
674 result = GetTrackingBranchViaGitConfig(
675 path, 'default', allow_broken_merge_settings=True, for_checkout=False)
677 if result is not None:
678 return StripRefsHeads(result[1], False)
680 raise OSError(errno.ENOENT,
681 "Manifest repository at %s is checked out to 'default', but "
682 'the git tracking configuration for that branch is broken; '
683 'failing due to that.' % (root,))
685 # pylint: disable=W0221
687 def Cached(cls, path, manifest_path=None, search=True):
688 """Return an instance, reusing an existing one if possible.
691 path: The pathway into a checkout; the root will be found automatically.
692 manifest_path: if given, the manifest.xml to use instead of the
693 checkouts internal manifest. Use with care.
694 search: If True, the path can point into the repo, and the root will
695 be found automatically. If False, the path *must* be the root, else
696 an OSError ENOENT will be thrown.
698 root, manifest_path = cls._NormalizeArgs(path, manifest_path,
701 md5 = cls._GetManifestHash(manifest_path)
702 obj, sources = cls._instance_cache.get((root, md5), (None, ()))
703 for include_target, target_md5 in sources:
704 if cls._GetManifestHash(include_target, True) != target_md5:
708 obj = cls(root, manifest_path=manifest_path)
709 sources = tuple((abspath, cls._GetManifestHash(abspath))
710 for (target, abspath) in obj.includes)
711 cls._instance_cache[(root, md5)] = (obj, sources)
715 def RunGit(git_repo, cmd, retry=True, **kwargs):
716 """RunCommand wrapper for git commands.
718 This suppresses print_cmd, and suppresses output by default. Git
719 functionality w/in this module should use this unless otherwise
720 warranted, to standardize git output (primarily, keeping it quiet
721 and being able to throw useful errors for it).
724 git_repo: Pathway to the git repo to operate on.
725 cmd: A sequence of the git subcommand to run. The 'git' prefix is
726 added automatically. If you wished to run 'git remote update',
727 this would be ['remote', 'update'] for example.
728 retry: If set, retry on transient errors. Defaults to True.
729 kwargs: Any RunCommand or GenericRetry options/overrides to use.
732 A CommandResult object.
735 def _ShouldRetry(exc):
736 """Returns True if push operation failed with a transient error."""
737 if (isinstance(exc, cros_build_lib.RunCommandError)
738 and exc.result and exc.result.error and
739 GIT_TRANSIENT_ERRORS_RE.search(exc.result.error)):
740 cros_build_lib.Warning('git reported transient error (cmd=%s); retrying',
741 cros_build_lib.CmdToStr(cmd), exc_info=True)
745 max_retry = kwargs.pop('max_retry', DEFAULT_RETRIES if retry else 0)
746 kwargs.setdefault('print_cmd', False)
747 kwargs.setdefault('sleep', DEFAULT_RETRY_INTERVAL)
748 kwargs.setdefault('cwd', git_repo)
749 kwargs.setdefault('capture_output', True)
750 return retry_util.GenericRetry(
751 _ShouldRetry, max_retry, cros_build_lib.RunCommand,
752 ['git'] + cmd, **kwargs)
755 def GetProjectUserEmail(git_repo):
756 """Get the email configured for the project."""
757 output = RunGit(git_repo, ['var', 'GIT_COMMITTER_IDENT']).output
758 m = re.search(r'<([^>]*)>', output.strip())
759 return m.group(1) if m else None
762 def MatchBranchName(git_repo, pattern, namespace=''):
763 """Return branches who match the specified regular expression.
766 git_repo: The git repository to operate upon.
767 pattern: The regexp to search with.
768 namespace: The namespace to restrict search to (e.g. 'refs/heads/').
771 List of matching branch names (with |namespace| trimmed).
773 match = re.compile(pattern, flags=re.I)
774 output = RunGit(git_repo, ['ls-remote', git_repo, namespace + '*']).output
775 branches = [x.split()[1] for x in output.splitlines()]
776 branches = [x[len(namespace):] for x in branches if x.startswith(namespace)]
777 return [x for x in branches if match.search(x)]
780 class AmbiguousBranchName(Exception):
781 """Error if given branch name matches too many branches."""
784 def MatchSingleBranchName(*args, **kwargs):
785 """Match exactly one branch name, else throw an exception.
788 See MatchBranchName for more details; all args are passed on.
794 raise AmbiguousBranchName if we did not match exactly one branch.
796 ret = MatchBranchName(*args, **kwargs)
798 raise AmbiguousBranchName('Did not match exactly 1 branch: %r' % ret)
802 def GetTrackingBranchViaGitConfig(git_repo, branch, for_checkout=True,
803 allow_broken_merge_settings=False,
805 """Pull the remote and upstream branch of a local branch
808 git_repo: The git repository to operate upon.
809 branch: The branch to inspect.
810 for_checkout: Whether to return localized refspecs, or the remote's
812 allow_broken_merge_settings: Repo in a couple of spots writes invalid
813 branch.mybranch.merge settings; if these are encountered, they're
814 normally treated as an error and this function returns None. If
815 this option is set to True, it suppresses this check.
816 recurse: If given and the target is local, then recurse through any
817 remote=. (aka locals). This is enabled by default, and is what allows
818 developers to have multiple local branches of development dependent
819 on one another; disabling this makes that work flow impossible,
820 thus disable it only with good reason. The value given controls how
821 deeply to recurse. Defaults to tracing through 10 levels of local
822 remotes. Disabling it is a matter of passing 0.
825 A tuple of the remote and the ref name of the tracking branch, or
826 None if it couldn't be found.
829 cmd = ['config', '--get-regexp',
830 r'branch\.%s\.(remote|merge)' % re.escape(branch)]
831 data = RunGit(git_repo, cmd).output.splitlines()
833 prefix = 'branch.%s.' % (branch,)
834 data = [x.split() for x in data]
835 vals = dict((x[0][len(prefix):], x[1]) for x in data)
837 if not allow_broken_merge_settings:
839 elif 'merge' not in vals:
840 # There isn't anything we can do here.
842 elif 'remote' not in vals:
843 # Repo v1.9.4 and up occasionally invalidly leave the remote out.
844 # Only occurs for the manifest repo fortunately.
845 vals['remote'] = 'origin'
846 remote, rev = vals['remote'], vals['merge']
847 # Suppress non branches; repo likes to write revisions and tags here,
848 # which is wrong (git hates it, nor will it honor it).
849 if rev.startswith('refs/remotes/'):
852 # We can't backtrack from here, or at least don't want to.
853 # This is likely refs/remotes/m/ which repo writes when dealing
854 # with a revision locked manifest.
856 if not rev.startswith('refs/heads/'):
857 # We explicitly don't allow pushing to tags, nor can one push
858 # to a sha1 remotely (makes no sense).
859 if not allow_broken_merge_settings:
864 'While tracing out tracking branches, we recursed too deeply: '
865 'bailing at %s' % branch)
866 return GetTrackingBranchViaGitConfig(
867 git_repo, StripRefsHeads(rev), for_checkout=for_checkout,
868 allow_broken_merge_settings=allow_broken_merge_settings,
871 rev = 'refs/remotes/%s/%s' % (remote, StripRefsHeads(rev))
873 except cros_build_lib.RunCommandError as e:
874 # 1 is the retcode for no matches.
875 if e.result.returncode != 1:
880 def GetTrackingBranchViaManifest(git_repo, for_checkout=True, for_push=False,
882 """Gets the appropriate push branch via the manifest if possible.
885 git_repo: The git repo to operate upon.
886 for_checkout: Whether to return localized refspecs, or the remote's
887 view of it. Note that depending on the remote, the remote may differ
888 if for_push is True or set to False.
889 for_push: Controls whether the remote and refspec returned is explicitly
891 manifest: A Manifest instance if one is available, else a
892 ManifestCheckout is created and used.
895 A tuple of a git target repo and the remote ref to push to, or
896 None if it couldnt be found. If for_checkout, then it returns
897 the localized version of it.
901 manifest = ManifestCheckout.Cached(git_repo)
903 checkout = manifest.FindCheckoutFromPath(git_repo, strict=False)
909 checkout.AssertPushable()
912 remote = checkout['push_remote']
914 remote = checkout['remote']
917 revision = checkout['tracking_branch']
919 revision = checkout['revision']
920 if not revision.startswith('refs/heads/'):
923 return remote, revision
924 except EnvironmentError as e:
925 if e.errno != errno.ENOENT:
930 def GetTrackingBranch(git_repo, branch=None, for_checkout=True, fallback=True,
931 manifest=None, for_push=False):
932 """Gets the appropriate push branch for the specified directory.
934 This function works on both repo projects and regular git checkouts.
937 1. We assume the manifest defined upstream is desirable.
938 2. No manifest? Assume tracking if configured is accurate.
939 3. If none of the above apply, you get 'origin', 'master' or None,
940 depending on fallback.
943 git_repo: Git repository to operate upon.
944 branch: Find the tracking branch for this branch. Defaults to the
945 current branch for |git_repo|.
946 for_checkout: Whether to return localized refspecs, or the remotes
948 fallback: If true and no remote/branch could be discerned, return
949 'origin', 'master'. If False, you get None.
950 Note that depending on the remote, the remote may differ
951 if for_push is True or set to False.
952 for_push: Controls whether the remote and refspec returned is explicitly
954 manifest: A Manifest instance if one is available, else a
955 ManifestCheckout is created and used.
958 A tuple of a git target repo and the remote ref to push to.
961 result = GetTrackingBranchViaManifest(git_repo, for_checkout=for_checkout,
962 manifest=manifest, for_push=for_push)
963 if result is not None:
967 branch = GetCurrentBranch(git_repo)
969 result = GetTrackingBranchViaGitConfig(git_repo, branch,
970 for_checkout=for_checkout)
971 if result is not None:
972 if (result[1].startswith('refs/heads/') or
973 result[1].startswith('refs/remotes/')):
979 return 'origin', 'refs/remotes/origin/master'
980 return 'origin', 'master'
983 def CreateBranch(git_repo, branch, branch_point='HEAD', track=False):
987 git_repo: Git repository to act on.
988 branch: Name of the branch to create.
989 branch_point: The ref to branch from. Defaults to 'HEAD'.
990 track: Whether to setup the branch to track its starting ref.
992 cmd = ['checkout', '-B', branch, branch_point]
994 cmd.append('--track')
995 RunGit(git_repo, cmd)
998 def GitPush(git_repo, refspec, push_to, dryrun=False, force=False, retry=True):
999 """Wrapper for pushing to a branch.
1002 git_repo: Git repository to act on.
1003 refspec: The local ref to push to the remote.
1004 push_to: A RemoteRef object representing the remote ref to push to.
1005 dryrun: Do not actually push anything. Uses the --dry-run option
1007 force: Whether to bypass non-fastforward checks.
1008 retry: Retry a push in case of transient errors.
1010 cmd = ['push', push_to.remote, '%s:%s' % (refspec, push_to.ref)]
1013 # The 'git push' command has --dry-run support built in, so leverage that.
1014 cmd.append('--dry-run')
1017 cmd.append('--force')
1019 RunGit(git_repo, cmd, retry=retry)
1022 # TODO(build): Switch callers of this function to use CreateBranch instead.
1023 def CreatePushBranch(branch, git_repo, sync=True, remote_push_branch=None):
1024 """Create a local branch for pushing changes inside a repo repository.
1027 branch: Local branch to create.
1028 git_repo: Git repository to create the branch in.
1029 sync: Update remote before creating push branch.
1030 remote_push_branch: A tuple of the (remote, branch) to push to. i.e.,
1031 ('cros', 'master'). By default it tries to
1032 automatically determine which tracking branch to use
1033 (see GetTrackingBranch()).
1035 if not remote_push_branch:
1036 remote, push_branch = GetTrackingBranch(git_repo, for_push=True)
1038 remote, push_branch = remote_push_branch
1041 cmd = ['remote', 'update', remote]
1042 RunGit(git_repo, cmd)
1044 RunGit(git_repo, ['checkout', '-B', branch, '-t', push_branch])
1047 def SyncPushBranch(git_repo, remote, rebase_target):
1048 """Sync and rebase a local push branch to the latest remote version.
1051 git_repo: Git repository to rebase in.
1052 remote: The remote returned by GetTrackingBranch(for_push=True)
1053 rebase_target: The branch name returned by GetTrackingBranch(). Must
1054 start with refs/remotes/ (specifically must be a proper remote
1055 target rather than an ambiguous name).
1057 if not rebase_target.startswith('refs/remotes/'):
1059 'Was asked to rebase to a non branch target w/in the push pathways. '
1060 'This is highly indicative of an internal bug. remote %s, rebase %s'
1061 % (remote, rebase_target))
1063 cmd = ['remote', 'update', remote]
1064 RunGit(git_repo, cmd)
1067 RunGit(git_repo, ['rebase', rebase_target])
1068 except cros_build_lib.RunCommandError:
1069 # Looks like our change conflicts with upstream. Cleanup our failed
1071 RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
1075 # TODO(build): Switch this to use the GitPush function.
1076 def PushWithRetry(branch, git_repo, dryrun=False, retries=5):
1077 """General method to push local git changes.
1079 This method only works with branches created via the CreatePushBranch
1083 branch: Local branch to push. Branch should have already been created
1084 with a local change committed ready to push to the remote branch. Must
1085 also already be checked out to that branch.
1086 git_repo: Git repository to push from.
1087 dryrun: Git push --dry-run if set to True.
1088 retries: The number of times to retry before giving up, default: 5
1091 GitPushFailed if push was unsuccessful after retries
1093 remote, ref = GetTrackingBranch(git_repo, branch, for_checkout=False,
1095 # Don't like invoking this twice, but there is a bit of API
1096 # impedence here; cros_mark_as_stable
1097 _, local_ref = GetTrackingBranch(git_repo, branch, for_push=True)
1099 if not ref.startswith('refs/heads/'):
1100 raise Exception('Was asked to push to a non branch namespace: %s' % (ref,))
1102 push_command = ['push', remote, '%s:%s' % (branch, ref)]
1103 cros_build_lib.Debug('Trying to push %s to %s:%s', git_repo, branch, ref)
1106 push_command.append('--dry-run')
1107 for retry in range(1, retries + 1):
1108 SyncPushBranch(git_repo, remote, local_ref)
1110 RunGit(git_repo, push_command)
1112 except cros_build_lib.RunCommandError:
1114 Warning('Error pushing changes trying again (%s/%s)', retry, retries)
1115 time.sleep(5 * retry)
1119 cros_build_lib.Info('Successfully pushed %s to %s:%s', git_repo, branch, ref)
1122 def CleanAndDetachHead(git_repo):
1123 """Remove all local changes and checkout a detached head.
1126 git_repo: Directory of git repository.
1128 RunGit(git_repo, ['am', '--abort'], error_code_ok=True)
1129 RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
1130 RunGit(git_repo, ['clean', '-dfx'])
1131 RunGit(git_repo, ['checkout', '--detach', '-f', 'HEAD'])
1134 def CleanAndCheckoutUpstream(git_repo, refresh_upstream=True):
1135 """Remove all local changes and checkout the latest origin.
1137 All local changes in the supplied repo will be removed. The branch will
1138 also be switched to a detached head pointing at the latest origin.
1141 git_repo: Directory of git repository.
1142 refresh_upstream: If True, run a remote update prior to checking it out.
1144 remote, local_upstream = GetTrackingBranch(git_repo,
1145 for_push=refresh_upstream)
1146 CleanAndDetachHead(git_repo)
1147 if refresh_upstream:
1148 RunGit(git_repo, ['remote', 'update', remote])
1149 RunGit(git_repo, ['checkout', local_upstream])
1152 def GetChromiteTrackingBranch():
1153 """Returns the remote branch associated with chromite."""
1154 cwd = os.path.dirname(os.path.realpath(__file__))
1155 result = GetTrackingBranch(cwd, for_checkout=False, fallback=False)
1157 _remote, branch = result
1158 if branch.startswith('refs/heads/'):
1160 return StripRefsHeads(branch)
1161 # Reaching here means it was refs/remotes/m/blah, or just plain invalid,
1162 # or that we're on a detached head in a repo not managed by chromite.
1164 # Manually try the manifest next.
1166 manifest = ManifestCheckout.Cached(cwd)
1167 # Ensure the manifest knows of this checkout.
1168 if manifest.FindCheckoutFromPath(cwd, strict=False):
1169 return manifest.manifest_branch
1170 except EnvironmentError as e:
1171 if e.errno != errno.ENOENT:
1174 # Not a manifest checkout.
1176 "Chromite checkout at %s isn't controlled by repo, nor is it on a "
1177 'branch (or if it is, the tracking configuration is missing or broken). '
1178 'Falling back to assuming the chromite checkout is derived from '
1179 "'master'; this *may* result in breakage." % cwd)
1183 def GarbageCollection(git_repo):
1184 """Cleanup unnecessary files and optimize the local repository.
1187 git_repo: Directory of git repository.
1189 # Use --auto so it only runs if housekeeping is necessary.
1190 RunGit(git_repo, ['gc', '--auto'])