1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Common functions for interacting with git and repo."""
12 # pylint: disable=W0402
18 # TODO(build): Fix this.
19 # This should be absolute import, but that requires fixing all
20 # relative imports first.
21 _path = os.path.realpath(__file__)
22 _path = os.path.normpath(os.path.join(os.path.dirname(_path), '..', '..'))
23 sys.path.insert(0, _path)
24 from chromite.buildbot import constants
25 from chromite.lib import cros_build_lib
26 from chromite.lib import osutils
27 from chromite.lib import retry_util
28 # Now restore it so that relative scripts don't get cranky.
32 # Retry a git operation if git returns a error response with any of these
33 # messages. It's all observed 'bad' GoB responses so far.
34 GIT_TRANSIENT_ERRORS = (
36 r'! \[remote rejected\].* -> .* \(error in hook\)',
39 r'! \[remote rejected\].* -> .* \(failed to lock\)',
42 r'! \[remote rejected\].* -> .* \(error in Gerrit backend\)',
45 r'remote error: Internal Server Error',
48 r'fatal: Couldn\'t find remote ref ',
51 r'git fetch_pack: expected ACK/NAK, got',
54 r'protocol error: bad pack header',
57 r'The remote end hung up unexpectedly',
60 r'TLS packet with unexpected length was received',
63 r'RPC failed; result=\d+, HTTP code = \d+',
66 r'The requested URL returned error: 500 while accessing',
69 GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS))
71 DEFAULT_RETRY_INTERVAL = 3
75 class RemoteRef(object):
76 """Object representing a remote ref.
78 A remote ref encapsulates both a remote (e.g., 'origin',
79 'https://chromium.googlesource.com/chromiumos/chromite.git', etc.) and a ref
80 name (e.g., 'refs/heads/master').
82 def __init__(self, remote, ref):
87 def FindRepoDir(path):
88 """Returns the nearest higher-level repo dir from the specified path.
91 path: The path to use. Defaults to cwd.
93 return osutils.FindInPathParents(
94 '.repo', path, test_func=os.path.isdir)
97 def FindRepoCheckoutRoot(path):
98 """Get the root of your repo managed checkout."""
99 repo_dir = FindRepoDir(path)
101 return os.path.dirname(repo_dir)
106 def IsSubmoduleCheckoutRoot(path, remote, url):
107 """Tests to see if a directory is the root of a git submodule checkout.
110 path: The directory to test.
111 remote: The remote to compare the |url| with.
112 url: The exact URL the |remote| needs to be pointed at.
114 if os.path.isdir(path):
115 remote_url = cros_build_lib.RunCommand(
116 ['git', '--git-dir', path, 'config', 'remote.%s.url' % remote],
117 redirect_stdout=True, debug_level=logging.DEBUG,
118 error_code_ok=True).output.strip()
119 if remote_url == url:
124 def ReinterpretPathForChroot(path):
125 """Returns reinterpreted path from outside the chroot for use inside.
128 path: The path to reinterpret. Must be in src tree.
130 root_path = os.path.join(FindRepoDir(path), '..')
132 path_abs_path = os.path.abspath(path)
133 root_abs_path = os.path.abspath(root_path)
135 # Strip the repository root from the path and strip first /.
136 relative_path = path_abs_path.replace(root_abs_path, '')[1:]
138 if relative_path == path_abs_path:
139 raise Exception('Error: path is outside your src tree, cannot reinterpret.')
141 new_path = os.path.join('/home', os.getenv('USER'), 'trunk', relative_path)
146 """Checks if there's a git repo rooted at a directory."""
147 return os.path.isdir(os.path.join(cwd, '.git'))
150 def IsGitRepositoryCorrupted(cwd):
151 """Verify that the specified git repository is not corrupted.
154 cwd: The git repository to verify.
157 True if the repository is corrupted.
159 cmd = ['fsck', '--no-progress', '--no-dangling']
163 except cros_build_lib.RunCommandError as ex:
164 logging.warn(str(ex))
168 _HEX_CHARS = frozenset(string.hexdigits)
169 def IsSHA1(value, full=True):
170 """Returns True if the given value looks like a sha1.
172 If full is True, then it must be full length- 40 chars. If False, >=6, and
175 if not all(x in _HEX_CHARS for x in value):
180 return l >= 6 and l <= 40
183 def IsRefsTags(value):
184 """Return True if the given value looks like a tag.
186 Currently this is identified via refs/tags/ prefixing.
188 return value.startswith("refs/tags/")
191 def GetGitRepoRevision(cwd, branch='HEAD'):
192 """Find the revision of a branch.
194 Defaults to current branch.
196 return RunGit(cwd, ['rev-parse', branch]).output.strip()
199 def DoesCommitExistInRepo(cwd, commit_hash):
200 """Determine if commit object exists in a repo.
203 cwd: A directory within the project repo.
204 commit_hash: The hash of the commit object to look for.
206 return 0 == RunGit(cwd, ['rev-list', '-n1', commit_hash],
207 error_code_ok=True).returncode
210 def DoesLocalBranchExist(repo_dir, branch):
211 """Returns True if the local branch exists.
214 repo_dir: Directory of the git repository to check.
215 branch: The name of the branch to test for.
217 return os.path.isfile(
218 os.path.join(repo_dir, '.git/refs/heads',
222 def GetCurrentBranch(cwd):
223 """Returns current branch of a repo, and None if repo is on detached HEAD."""
225 ret = RunGit(cwd, ['symbolic-ref', '-q', 'HEAD'])
226 return StripRefsHeads(ret.output.strip(), False)
227 except cros_build_lib.RunCommandError as e:
228 if e.result.returncode != 1:
233 def StripRefsHeads(ref, strict=True):
234 """Remove leading 'refs/heads/' from a ref name.
236 If strict is True, an Exception is thrown if the ref doesn't start with
237 refs/heads. If strict is False, the original ref is returned.
239 if not ref.startswith('refs/heads/') and strict:
240 raise Exception('Ref name %s does not start with refs/heads/' % ref)
242 return ref.replace('refs/heads/', '')
246 """Remove leading 'refs/heads', 'refs/remotes/[^/]+/' from a ref name."""
247 ref = StripRefsHeads(ref, False)
248 if ref.startswith("refs/remotes/"):
249 return ref.split("/", 3)[-1]
253 def NormalizeRef(ref):
254 """Convert git branch refs into fully qualified form."""
255 if ref and not ref.startswith('refs/'):
256 ref = 'refs/heads/%s' % ref
260 def NormalizeRemoteRef(remote, ref):
261 """Convert git branch refs into fully qualified remote form."""
263 # Support changing local ref to remote ref, or changing the remote
267 if not ref.startswith('refs/'):
268 ref = 'refs/remotes/%s/%s' % (remote, ref)
273 class ProjectCheckout(dict):
274 """Attributes of a given project in the manifest checkout.
276 TODO(davidjames): Convert this into an ordinary object instead of a dict.
279 def __init__(self, attrs):
283 attrs: The attributes associated with this checkout, as a dictionary.
285 dict.__init__(self, attrs)
287 def AssertPushable(self):
288 """Verify that it is safe to push changes to this repository."""
289 if not self['pushable']:
290 remote = self['remote']
291 raise AssertionError('Remote %s is not pushable.' % (remote,))
293 def IsBranchableProject(self):
294 """Return whether this project is hosted on ChromeOS git servers."""
295 return (self['remote'] in constants.CROS_REMOTES and
296 re.match(constants.BRANCHABLE_PROJECTS[self['remote']], self['name']))
298 def IsPatchable(self):
299 """Returns whether this project is patchable.
301 For projects that get checked out at multiple paths and/or branches,
302 this method can be used to determine which project path a patch
303 should be applied to.
306 True if the project corresponding to the checkout is patchable.
308 # There are 2 ways we determine if a project is patchable.
309 # - For an unversioned manifest, if the 'revision' is a raw SHA1 hash
310 # and not a named branch, assume it is a pinned project path and can not
312 # - For a versioned manifest (generated via repo -r), repo will sets
313 # revision to the actual git sha1 ref, and adds an 'upstream'
314 # field corresponding to branch name in the original manifest. For
315 # a project with a SHA1 'revision' but no named branch in the
316 # 'upstream' field, assume it can not be patched.
317 return not IsSHA1(self.get('upstream', self['revision']))
319 def GetPath(self, absolute=False):
320 """Get the path to the checkout.
323 absolute: If True, return an absolute path. If False,
324 return a path relative to the repo root.
326 return self['local_path'] if absolute else self['path']
329 class Manifest(object):
330 """SAX handler that parses the manifest document.
333 checkouts_by_name: A dictionary mapping the names for <project> tags to a
334 list of ProjectCheckout objects.
335 checkouts_by_path: A dictionary mapping paths for <project> tags to a single
336 ProjectCheckout object.
337 default: The attributes of the <default> tag.
338 includes: A list of XML files that should be pulled in to the manifest.
339 These includes are represented as a list of (name, path) tuples.
340 manifest_include_dir: If given, this is where to start looking for
342 projects: DEPRECATED. A dictionary mapping the names for <project> tags to
343 a single ProjectCheckout object. This is now deprecated, since each
344 project can map to multiple ProjectCheckout objects.
345 remotes: A dictionary mapping <remote> tags to the associated attributes.
346 revision: The revision of the manifest repository. If not specified, this
352 def __init__(self, source, manifest_include_dir=None):
353 """Initialize this instance.
356 source: The path to the manifest to parse. May be a file handle.
357 manifest_include_dir: If given, this is where to start looking for
362 self.checkouts_by_path = {}
363 self.checkouts_by_name = {}
367 self.manifest_include_dir = manifest_include_dir
368 self._RunParser(source)
369 self.includes = tuple(self.includes)
371 def _RunParser(self, source, finalize=True):
372 parser = sax.make_parser()
373 handler = sax.handler.ContentHandler()
374 handler.startElement = self._ProcessElement
375 parser.setContentHandler(handler)
378 self._FinalizeAllProjectData()
380 def _ProcessElement(self, name, attrs):
381 """Stores the default manifest properties and per-project overrides."""
382 attrs = dict(attrs.items())
383 if name == 'default':
385 elif name == 'remote':
386 attrs.setdefault('alias', attrs['name'])
387 self.remotes[attrs['name']] = attrs
388 elif name == 'project':
389 self.checkouts_by_path[attrs['path']] = attrs
390 self.checkouts_by_name.setdefault(attrs['name'], []).append(attrs)
391 elif name == 'manifest':
392 self.revision = attrs.get('revision')
393 elif name == 'include':
394 if self.manifest_include_dir is None:
396 errno.ENOENT, "No manifest_include_dir given, but an include was "
397 "encountered; attrs=%r" % (attrs,))
398 # Include is calculated relative to the manifest that has the include;
399 # thus set the path temporarily to the dirname of the target.
400 original_include_dir = self.manifest_include_dir
401 include_path = os.path.realpath(
402 os.path.join(original_include_dir, attrs['name']))
403 self.includes.append((attrs['name'], include_path))
404 self._RunParser(include_path, finalize=False)
406 def _FinalizeAllProjectData(self):
407 """Rewrite projects mixing defaults in and adding our attributes."""
408 for path_data in self.checkouts_by_path.itervalues():
409 self._FinalizeProjectData(path_data)
411 def _FinalizeProjectData(self, attrs):
412 """Sets up useful properties for a project.
415 attrs: The attribute dictionary of a <project> tag.
417 for key in ('remote', 'revision'):
418 attrs.setdefault(key, self.default.get(key))
420 remote = attrs['remote']
421 assert remote in self.remotes
422 remote_name = attrs['remote_alias'] = self.remotes[remote]['alias']
424 # 'repo manifest -r' adds an 'upstream' attribute to the project tag for the
425 # manifests it generates. We can use the attribute to get a valid branch
426 # instead of a sha1 for these types of manifests.
427 upstream = attrs.get('upstream', attrs['revision'])
429 # The current version of repo we use has a bug: When you create a new
430 # repo checkout from a revlocked manifest, the 'upstream' attribute will
431 # just point at a SHA1. The default revision will still be correct,
432 # however. For now, return the default revision as our best guess as to
433 # what the upstream branch for this repository would be. This guess may
434 # sometimes be wrong, but it's correct for all of the repositories where
435 # we need to push changes (e.g., the overlays).
436 # TODO(davidjames): Either fix the repo bug, or update our logic here to
437 # check the manifest repository to find the right tracking branch.
438 upstream = self.default.get('revision', 'refs/heads/master')
440 attrs['tracking_branch'] = 'refs/remotes/%s/%s' % (
441 remote_name, StripRefs(upstream),
444 attrs['pushable'] = remote in constants.GIT_REMOTES
445 if attrs['pushable']:
446 attrs['push_remote'] = remote
447 attrs['push_remote_url'] = constants.GIT_REMOTES[remote]
448 attrs['push_url'] = '%s/%s' % (attrs['push_remote_url'], attrs['name'])
449 groups = set(attrs.get('groups', 'default').replace(',', ' ').split())
450 groups.add('default')
451 attrs['groups'] = frozenset(groups)
453 # Compute the local ref space.
454 # Sanitize a couple path fragments to simplify assumptions in this
455 # class, and in consuming code.
456 attrs.setdefault('path', attrs['name'])
457 for key in ('name', 'path'):
458 attrs[key] = os.path.normpath(attrs[key])
461 def _GetManifestHash(source, ignore_missing=False):
462 if isinstance(source, basestring):
464 # TODO(build): convert this to osutils.ReadFile once these
465 # classes are moved out into their own module (if possible;
466 # may still be cyclic).
467 with open(source, 'rb') as f:
468 return hashlib.md5(f.read()).hexdigest()
469 except EnvironmentError as e:
470 if e.errno != errno.ENOENT or not ignore_missing:
473 md5 = hashlib.md5(source.read()).hexdigest()
478 def Cached(cls, source, manifest_include_dir=None):
479 """Return an instance, reusing an existing one if possible.
481 May be a seekable filehandle, or a filepath.
482 See __init__ for an explanation of these arguments.
485 md5 = cls._GetManifestHash(source)
486 obj, sources = cls._instance_cache.get(md5, (None, ()))
487 if manifest_include_dir is None and sources:
488 # We're being invoked in a different way than the orignal
489 # caching; disregard the cached entry.
490 # Most likely, the instantiation will explode; let it fly.
491 obj, sources = None, ()
492 for include_target, target_md5 in sources:
493 if cls._GetManifestHash(include_target, True) != target_md5:
497 obj = cls(source, manifest_include_dir=manifest_include_dir)
498 sources = tuple((abspath, cls._GetManifestHash(abspath))
499 for (target, abspath) in obj.includes)
500 cls._instance_cache[md5] = (obj, sources)
505 class ManifestCheckout(Manifest):
506 """A Manifest Handler for a specific manifest checkout."""
510 # pylint: disable=W0221
511 def __init__(self, path, manifest_path=None, search=True):
512 """Initialize this instance.
515 path: Path into a manifest checkout (doesn't have to be the root).
516 manifest_path: If supplied, the manifest to use. Else the manifest
517 in the root of the checkout is used. May be a seekable file handle.
518 search: If True, the path can point into the repo, and the root will
519 be found automatically. If False, the path *must* be the root, else
520 an OSError ENOENT will be thrown.
523 OSError: if a failure occurs.
525 self.root, manifest_path = self._NormalizeArgs(
526 path, manifest_path, search=search)
528 self.manifest_path = os.path.realpath(manifest_path)
529 manifest_include_dir = os.path.dirname(self.manifest_path)
530 self.manifest_branch = self._GetManifestsBranch(self.root)
531 self._content_merging = {}
532 self.configured_groups = self._GetManifestGroups(self.root)
533 Manifest.__init__(self, self.manifest_path,
534 manifest_include_dir=manifest_include_dir)
537 def _NormalizeArgs(path, manifest_path=None, search=True):
538 root = FindRepoCheckoutRoot(path)
540 raise OSError(errno.ENOENT, "Couldn't find repo root: %s" % (path,))
541 root = os.path.normpath(os.path.realpath(root))
543 if os.path.normpath(os.path.realpath(path)) != root:
544 raise OSError(errno.ENOENT, "Path %s is not a repo root, and search "
545 "is disabled." % path)
546 if manifest_path is None:
547 manifest_path = os.path.join(root, '.repo', 'manifest.xml')
548 return root, manifest_path
550 def ProjectIsContentMerging(self, project):
551 """Returns whether the given project has content merging enabled in git.
553 Note this functionality should *only* be used against a remote that is
554 known to be >=gerrit-2.2; <gerrit-2.2 lacks the required branch holding
555 this data thus will trigger a RunCommandError.
558 True if content merging is enabled.
561 AssertionError: If no patchable checkout was found or if the patchable
562 checkout does not have a pushable project remote.
563 RunCommandError: If the branch can't be fetched due to network
564 conditions or if this was invoked against a <gerrit-2.2 server,
565 or a mirror that has refs/meta/config stripped from it.
567 result = self._content_merging.get(project)
569 checkouts = self.FindCheckouts(project, only_patchable=True)
570 if len(checkouts) < 1:
571 raise AssertionError('No patchable checkout of %s was found' % project)
572 for checkout in checkouts:
573 checkout.AssertPushable()
574 self._content_merging[project] = result = _GitRepoIsContentMerging(
575 checkout['local_path'], checkout['push_remote'])
578 def FindCheckouts(self, project, branch=None, only_patchable=False):
579 """Returns the list of checkouts for a given |project|/|branch|.
582 project: Project name to search for.
583 branch: Branch to use.
584 only_patchable: Restrict search to patchable project paths.
587 A list of ProjectCheckout objects.
590 for checkout in self.checkouts_by_name.get(project, []):
591 if project == checkout['name']:
592 if only_patchable and not checkout.IsPatchable():
594 tracking_branch = checkout['tracking_branch']
595 if branch is None or StripRefs(branch) == StripRefs(tracking_branch):
596 checkouts.append(checkout)
599 def FindCheckout(self, project, branch=None, strict=True):
600 """Returns the checkout associated with a given project/branch.
603 project: The project to look for.
604 branch: The branch that the project is tracking.
605 strict: Raise AssertionError if a checkout cannot be found.
608 A ProjectCheckout object.
611 AssertionError if there is more than one checkout associated with the
612 given project/branch combination.
614 checkouts = self.FindCheckouts(project, branch)
615 if len(checkouts) < 1:
617 raise AssertionError('Could not find checkout of %s' % (project,))
619 elif len(checkouts) > 1:
620 raise AssertionError('Too many checkouts found for %s' % project)
623 def ListCheckouts(self):
624 """List the checkouts in the manifest.
627 A list of ProjectCheckout objects.
629 return self.checkouts_by_path.values()
631 def FindCheckoutFromPath(self, path, strict=True):
632 """Find the associated checkouts for a given |path|.
634 The |path| can either be to the root of a project, or within the
635 project itself (chromite/buildbot for example). It may be relative
636 to the repo root, or an absolute path. If |path| is not within a
637 checkout, return None.
640 path: Path to examine.
641 strict: If True, fail when no checkout is found.
644 None if no checkout is found, else the checkout.
646 # Realpath everything sans the target to keep people happy about
647 # how symlinks are handled; exempt the final node since following
648 # through that is unlikely even remotely desired.
649 tmp = os.path.join(self.root, os.path.dirname(path))
650 path = os.path.join(os.path.realpath(tmp), os.path.basename(path))
651 path = os.path.normpath(path) + '/'
653 for checkout in self.ListCheckouts():
654 if path.startswith(checkout['local_path'] + '/'):
655 candidates.append((checkout['path'], checkout))
659 raise AssertionError('Could not find repo project at %s' % (path,))
662 # The checkout with the greatest common path prefix is the owner of
663 # the given pathway. Return that.
664 return max(candidates)[1]
666 def _FinalizeAllProjectData(self):
667 """Rewrite projects mixing defaults in and adding our attributes."""
668 Manifest._FinalizeAllProjectData(self)
669 for key, value in self.checkouts_by_path.iteritems():
670 self.checkouts_by_path[key] = ProjectCheckout(value)
671 for key, value in self.checkouts_by_name.iteritems():
672 self.checkouts_by_name[key] = \
673 [ProjectCheckout(x) for x in value]
675 def _FinalizeProjectData(self, attrs):
676 Manifest._FinalizeProjectData(self, attrs)
677 attrs['local_path'] = os.path.join(self.root, attrs['path'])
680 def _GetManifestGroups(root):
681 """Discern which manifest groups were enabled for this checkout."""
682 path = os.path.join(root, '.repo', 'manifests.git')
684 result = RunGit(path, ['config', '--get', 'manifest.groups'])
685 except cros_build_lib.RunCommandError as e:
686 if e.result.returncode == 1:
687 # Value wasn't found, which is fine.
688 return frozenset(['default'])
689 # If exit code 2, multiple values matched (broken checkout). Anything
690 # else, git internal error.
693 result = result.output.replace(',', ' ').split()
696 return frozenset(result)
699 def _GetManifestsBranch(root):
700 """Get the tracking branch of the manifest repository.
705 # Suppress the normal "if it ain't refs/heads, we don't want none o' that"
706 # check for the merge target; repo writes the ambigious form of the branch
707 # target for `repo init -u url -b some-branch` usages (aka, 'master'
708 # instead of 'refs/heads/master').
709 path = os.path.join(root, '.repo', 'manifests')
710 current_branch = GetCurrentBranch(path)
711 if current_branch != 'default':
712 raise OSError(errno.ENOENT,
713 "Manifest repository at %s is checked out to %s. "
714 "It should be checked out to 'default'."
715 % (root, 'detached HEAD' if current_branch is None
716 else current_branch))
718 result = GetTrackingBranchViaGitConfig(
719 path, 'default', allow_broken_merge_settings=True, for_checkout=False)
721 if result is not None:
722 return StripRefsHeads(result[1], False)
724 raise OSError(errno.ENOENT,
725 "Manifest repository at %s is checked out to 'default', but "
726 "the git tracking configuration for that branch is broken; "
727 "failing due to that." % (root,))
729 # pylint: disable=W0221
731 def Cached(cls, path, manifest_path=None, search=True):
732 """Return an instance, reusing an existing one if possible.
735 path: The pathway into a checkout; the root will be found automatically.
736 manifest_path: if given, the manifest.xml to use instead of the
737 checkouts internal manifest. Use with care.
738 search: If True, the path can point into the repo, and the root will
739 be found automatically. If False, the path *must* be the root, else
740 an OSError ENOENT will be thrown.
742 root, manifest_path = cls._NormalizeArgs(path, manifest_path,
745 md5 = cls._GetManifestHash(manifest_path)
746 obj, sources = cls._instance_cache.get((root, md5), (None, ()))
747 for include_target, target_md5 in sources:
748 if cls._GetManifestHash(include_target, True) != target_md5:
752 obj = cls(root, manifest_path=manifest_path)
753 sources = tuple((abspath, cls._GetManifestHash(abspath))
754 for (target, abspath) in obj.includes)
755 cls._instance_cache[(root, md5)] = (obj, sources)
759 def _GitRepoIsContentMerging(git_repo, remote):
760 """Identify if the given git repo has content merging marked.
762 This is a gerrit >=2.2 bit of functionality; specifically, the content
763 merging configuration is stored in a specially crafted branch which
764 we access. If the branch is fetchable, we either return True or False.
767 git_repo: The local path to the git repository to inspect.
768 remote: The configured remote to use from the given git repository.
771 True if content merging is enabled, False if not.
774 RunCommandError: Thrown if fetching fails due to either the namespace
775 not existing, or a network error intervening.
777 # Need to use the manifest to get upstream gerrit; also, if upstream
778 # doesn't provide a refs/meta/config for the repo, this will fail.
779 RunGit(git_repo, ['fetch', remote, 'refs/meta/config:refs/meta/config'])
781 content = RunGit(git_repo, ['show', 'refs/meta/config:project.config'],
784 if content.returncode != 0:
788 result = RunGit(git_repo, ['config', '-f', '/dev/stdin', '--get',
789 'submit.mergeContent'], input=content.output)
790 return result.output.strip().lower() == 'true'
791 except cros_build_lib.RunCommandError as e:
792 # If the field isn't set at all, exit code is 1.
793 # Anything else is a bad invocation or an indecipherable state.
794 if e.result.returncode != 1:
800 def RunGit(git_repo, cmd, retry=True, **kwargs):
801 """RunCommand wrapper for git commands.
803 This suppresses print_cmd, and suppresses output by default. Git
804 functionality w/in this module should use this unless otherwise
805 warranted, to standardize git output (primarily, keeping it quiet
806 and being able to throw useful errors for it).
809 git_repo: Pathway to the git repo to operate on.
810 cmd: A sequence of the git subcommand to run. The 'git' prefix is
811 added automatically. If you wished to run 'git remote update',
812 this would be ['remote', 'update'] for example.
813 retry: If set, retry on transient errors. Defaults to True.
814 kwargs: Any RunCommand or GenericRetry options/overrides to use.
817 A CommandResult object.
820 def _ShouldRetry(exc):
821 """Returns True if push operation failed with a transient error."""
822 if (isinstance(exc, cros_build_lib.RunCommandError)
823 and exc.result and exc.result.error and
824 GIT_TRANSIENT_ERRORS_RE.search(exc.result.error)):
825 cros_build_lib.Warning('git reported transient error (cmd=%s); retrying',
826 cros_build_lib.CmdToStr(cmd), exc_info=True)
830 max_retry = kwargs.pop('max_retry', DEFAULT_RETRIES if retry else 0)
831 kwargs.setdefault('print_cmd', False)
832 kwargs.setdefault('sleep', DEFAULT_RETRY_INTERVAL)
833 kwargs.setdefault('cwd', git_repo)
834 kwargs.setdefault('capture_output', True)
835 return retry_util.GenericRetry(
836 _ShouldRetry, max_retry, cros_build_lib.RunCommand,
837 ['git'] + cmd, **kwargs)
840 def GetProjectUserEmail(git_repo):
841 """Get the email configured for the project."""
842 output = RunGit(git_repo, ['var', 'GIT_COMMITTER_IDENT']).output
843 m = re.search(r'<([^>]*)>', output.strip())
844 return m.group(1) if m else None
847 def MatchBranchName(git_repo, pattern, namespace=''):
848 """Return branches who match the specified regular expression.
851 git_repo: The git repository to operate upon.
852 pattern: The regexp to search with.
853 namespace: The namespace to restrict search to (e.g. 'refs/heads/').
856 List of matching branch names (with |namespace| trimmed).
858 match = re.compile(pattern, flags=re.I)
859 output = RunGit(git_repo, ['ls-remote', git_repo, namespace + '*']).output
860 branches = [x.split()[1] for x in output.splitlines()]
861 branches = [x[len(namespace):] for x in branches if x.startswith(namespace)]
862 return [x for x in branches if match.search(x)]
865 class AmbiguousBranchName(Exception):
866 """Error if given branch name matches too many branches."""
869 def MatchSingleBranchName(*args, **kwargs):
870 """Match exactly one branch name, else throw an exception.
873 See MatchBranchName for more details; all args are passed on.
879 raise AmbiguousBranchName if we did not match exactly one branch.
881 ret = MatchBranchName(*args, **kwargs)
883 raise AmbiguousBranchName('Did not match exactly 1 branch: %r' % ret)
887 def GetTrackingBranchViaGitConfig(git_repo, branch, for_checkout=True,
888 allow_broken_merge_settings=False,
890 """Pull the remote and upstream branch of a local branch
893 git_repo: The git repository to operate upon.
894 branch: The branch to inspect.
895 for_checkout: Whether to return localized refspecs, or the remote's
897 allow_broken_merge_settings: Repo in a couple of spots writes invalid
898 branch.mybranch.merge settings; if these are encountered, they're
899 normally treated as an error and this function returns None. If
900 this option is set to True, it suppresses this check.
901 recurse: If given and the target is local, then recurse through any
902 remote=. (aka locals). This is enabled by default, and is what allows
903 developers to have multiple local branches of development dependent
904 on one another; disabling this makes that work flow impossible,
905 thus disable it only with good reason. The value given controls how
906 deeply to recurse. Defaults to tracing through 10 levels of local
907 remotes. Disabling it is a matter of passing 0.
910 A tuple of the remote and the ref name of the tracking branch, or
911 None if it couldn't be found.
914 cmd = ['config', '--get-regexp',
915 r'branch\.%s\.(remote|merge)' % re.escape(branch)]
916 data = RunGit(git_repo, cmd).output.splitlines()
918 prefix = 'branch.%s.' % (branch,)
919 data = [x.split() for x in data]
920 vals = dict((x[0][len(prefix):], x[1]) for x in data)
922 if not allow_broken_merge_settings:
924 elif 'merge' not in vals:
925 # There isn't anything we can do here.
927 elif 'remote' not in vals:
928 # Repo v1.9.4 and up occasionally invalidly leave the remote out.
929 # Only occurs for the manifest repo fortunately.
930 vals['remote'] = 'origin'
931 remote, rev = vals['remote'], vals['merge']
932 # Suppress non branches; repo likes to write revisions and tags here,
933 # which is wrong (git hates it, nor will it honor it).
934 if rev.startswith('refs/remotes/'):
937 # We can't backtrack from here, or at least don't want to.
938 # This is likely refs/remotes/m/ which repo writes when dealing
939 # with a revision locked manifest.
941 if not rev.startswith('refs/heads/'):
942 # We explicitly don't allow pushing to tags, nor can one push
943 # to a sha1 remotely (makes no sense).
944 if not allow_broken_merge_settings:
949 "While tracing out tracking branches, we recursed too deeply: "
950 "bailing at %s" % branch)
951 return GetTrackingBranchViaGitConfig(
952 git_repo, StripRefsHeads(rev), for_checkout=for_checkout,
953 allow_broken_merge_settings=allow_broken_merge_settings,
956 rev = 'refs/remotes/%s/%s' % (remote, StripRefsHeads(rev))
958 except cros_build_lib.RunCommandError as e:
959 # 1 is the retcode for no matches.
960 if e.result.returncode != 1:
965 def GetTrackingBranchViaManifest(git_repo, for_checkout=True, for_push=False,
967 """Gets the appropriate push branch via the manifest if possible.
970 git_repo: The git repo to operate upon.
971 for_checkout: Whether to return localized refspecs, or the remote's
972 view of it. Note that depending on the remote, the remote may differ
973 if for_push is True or set to False.
974 for_push: Controls whether the remote and refspec returned is explicitly
976 manifest: A Manifest instance if one is available, else a
977 ManifestCheckout is created and used.
980 A tuple of a git target repo and the remote ref to push to, or
981 None if it couldnt be found. If for_checkout, then it returns
982 the localized version of it.
986 manifest = ManifestCheckout.Cached(git_repo)
988 checkout = manifest.FindCheckoutFromPath(git_repo, strict=False)
994 checkout.AssertPushable()
997 remote = checkout['push_remote']
999 remote = checkout['remote']
1002 revision = checkout['tracking_branch']
1004 revision = checkout['revision']
1005 if not revision.startswith('refs/heads/'):
1008 return remote, revision
1009 except EnvironmentError as e:
1010 if e.errno != errno.ENOENT:
1015 def GetTrackingBranch(git_repo, branch=None, for_checkout=True, fallback=True,
1016 manifest=None, for_push=False):
1017 """Gets the appropriate push branch for the specified directory.
1019 This function works on both repo projects and regular git checkouts.
1022 1. We assume the manifest defined upstream is desirable.
1023 2. No manifest? Assume tracking if configured is accurate.
1024 3. If none of the above apply, you get 'origin', 'master' or None,
1025 depending on fallback.
1028 git_repo: Git repository to operate upon.
1029 branch: Find the tracking branch for this branch. Defaults to the
1030 current branch for |git_repo|.
1031 for_checkout: Whether to return localized refspecs, or the remotes
1033 fallback: If true and no remote/branch could be discerned, return
1034 'origin', 'master'. If False, you get None.
1035 Note that depending on the remote, the remote may differ
1036 if for_push is True or set to False.
1037 for_push: Controls whether the remote and refspec returned is explicitly
1039 manifest: A Manifest instance if one is available, else a
1040 ManifestCheckout is created and used.
1043 A tuple of a git target repo and the remote ref to push to.
1046 result = GetTrackingBranchViaManifest(git_repo, for_checkout=for_checkout,
1047 manifest=manifest, for_push=for_push)
1048 if result is not None:
1052 branch = GetCurrentBranch(git_repo)
1054 result = GetTrackingBranchViaGitConfig(git_repo, branch,
1055 for_checkout=for_checkout)
1056 if result is not None:
1057 if (result[1].startswith('refs/heads/') or
1058 result[1].startswith('refs/remotes/')):
1064 return 'origin', 'refs/remotes/origin/master'
1065 return 'origin', 'master'
1068 def CreateBranch(git_repo, branch, branch_point='HEAD', track=False):
1072 git_repo: Git repository to act on.
1073 branch: Name of the branch to create.
1074 branch_point: The ref to branch from. Defaults to 'HEAD'.
1075 track: Whether to setup the branch to track its starting ref.
1077 cmd = ['checkout', '-B', branch, branch_point]
1079 cmd.append('--track')
1080 RunGit(git_repo, cmd)
1083 def GitPush(git_repo, refspec, push_to, dryrun=False, force=False, retry=True):
1084 """Wrapper for pushing to a branch.
1087 git_repo: Git repository to act on.
1088 refspec: The local ref to push to the remote.
1089 push_to: A RemoteRef object representing the remote ref to push to.
1090 dryrun: Do not actually push anything. Uses the --dry-run option
1092 force: Whether to bypass non-fastforward checks.
1093 retry: Retry a push in case of transient errors.
1095 cmd = ['push', push_to.remote, '%s:%s' % (refspec, push_to.ref)]
1098 # The 'git push' command has --dry-run support built in, so leverage that.
1099 cmd.append('--dry-run')
1102 cmd.append('--force')
1104 RunGit(git_repo, cmd, retry=retry)
1107 # TODO(build): Switch callers of this function to use CreateBranch instead.
1108 def CreatePushBranch(branch, git_repo, sync=True, remote_push_branch=None):
1109 """Create a local branch for pushing changes inside a repo repository.
1112 branch: Local branch to create.
1113 git_repo: Git repository to create the branch in.
1114 sync: Update remote before creating push branch.
1115 remote_push_branch: A tuple of the (remote, branch) to push to. i.e.,
1116 ('cros', 'master'). By default it tries to
1117 automatically determine which tracking branch to use
1118 (see GetTrackingBranch()).
1120 if not remote_push_branch:
1121 remote, push_branch = GetTrackingBranch(git_repo, for_push=True)
1123 remote, push_branch = remote_push_branch
1126 cmd = ['remote', 'update', remote]
1127 RunGit(git_repo, cmd)
1129 RunGit(git_repo, ['checkout', '-B', branch, '-t', push_branch])
1132 def SyncPushBranch(git_repo, remote, rebase_target):
1133 """Sync and rebase a local push branch to the latest remote version.
1136 git_repo: Git repository to rebase in.
1137 remote: The remote returned by GetTrackingBranch(for_push=True)
1138 rebase_target: The branch name returned by GetTrackingBranch(). Must
1139 start with refs/remotes/ (specifically must be a proper remote
1140 target rather than an ambiguous name).
1142 if not rebase_target.startswith("refs/remotes/"):
1144 "Was asked to rebase to a non branch target w/in the push pathways. "
1145 "This is highly indicative of an internal bug. remote %s, rebase %s"
1146 % (remote, rebase_target))
1148 cmd = ['remote', 'update', remote]
1149 RunGit(git_repo, cmd)
1152 RunGit(git_repo, ['rebase', rebase_target])
1153 except cros_build_lib.RunCommandError:
1154 # Looks like our change conflicts with upstream. Cleanup our failed
1156 RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
1160 # TODO(build): Switch this to use the GitPush function.
1161 def PushWithRetry(branch, git_repo, dryrun=False, retries=5):
1162 """General method to push local git changes.
1164 This method only works with branches created via the CreatePushBranch
1168 branch: Local branch to push. Branch should have already been created
1169 with a local change committed ready to push to the remote branch. Must
1170 also already be checked out to that branch.
1171 git_repo: Git repository to push from.
1172 dryrun: Git push --dry-run if set to True.
1173 retries: The number of times to retry before giving up, default: 5
1176 GitPushFailed if push was unsuccessful after retries
1178 remote, ref = GetTrackingBranch(git_repo, branch, for_checkout=False,
1180 # Don't like invoking this twice, but there is a bit of API
1181 # impedence here; cros_mark_as_stable
1182 _, local_ref = GetTrackingBranch(git_repo, branch, for_push=True)
1184 if not ref.startswith("refs/heads/"):
1185 raise Exception("Was asked to push to a non branch namespace: %s" % (ref,))
1187 push_command = ['push', remote, '%s:%s' % (branch, ref)]
1188 cros_build_lib.Debug("Trying to push %s to %s:%s", git_repo, branch, ref)
1191 push_command.append('--dry-run')
1192 for retry in range(1, retries + 1):
1193 SyncPushBranch(git_repo, remote, local_ref)
1195 RunGit(git_repo, push_command)
1197 except cros_build_lib.RunCommandError:
1199 Warning('Error pushing changes trying again (%s/%s)', retry, retries)
1200 time.sleep(5 * retry)
1204 cros_build_lib.Info("Successfully pushed %s to %s:%s", git_repo, branch, ref)
1207 def CleanAndDetachHead(git_repo):
1208 """Remove all local changes and checkout a detached head.
1211 git_repo: Directory of git repository.
1213 RunGit(git_repo, ['am', '--abort'], error_code_ok=True)
1214 RunGit(git_repo, ['rebase', '--abort'], error_code_ok=True)
1215 RunGit(git_repo, ['clean', '-dfx'])
1216 RunGit(git_repo, ['checkout', '--detach', '-f', 'HEAD'])
1219 def CleanAndCheckoutUpstream(git_repo, refresh_upstream=True):
1220 """Remove all local changes and checkout the latest origin.
1222 All local changes in the supplied repo will be removed. The branch will
1223 also be switched to a detached head pointing at the latest origin.
1226 git_repo: Directory of git repository.
1227 refresh_upstream: If True, run a remote update prior to checking it out.
1229 remote, local_upstream = GetTrackingBranch(git_repo,
1230 for_push=refresh_upstream)
1231 CleanAndDetachHead(git_repo)
1232 if refresh_upstream:
1233 RunGit(git_repo, ['remote', 'update', remote])
1234 RunGit(git_repo, ['checkout', local_upstream])
1237 def GetChromiteTrackingBranch():
1238 """Returns the remote branch associated with chromite."""
1239 cwd = os.path.dirname(os.path.realpath(__file__))
1240 result = GetTrackingBranch(cwd, for_checkout=False, fallback=False)
1242 _remote, branch = result
1243 if branch.startswith('refs/heads/'):
1245 return StripRefsHeads(branch)
1246 # Reaching here means it was refs/remotes/m/blah, or just plain invalid,
1247 # or that we're on a detached head in a repo not managed by chromite.
1249 # Manually try the manifest next.
1251 manifest = ManifestCheckout.Cached(cwd)
1252 # Ensure the manifest knows of this checkout.
1253 if manifest.FindCheckoutFromPath(cwd, strict=False):
1254 return manifest.manifest_branch
1255 except EnvironmentError as e:
1256 if e.errno != errno.ENOENT:
1259 # Not a manifest checkout.
1261 "Chromite checkout at %s isn't controlled by repo, nor is it on a "
1262 "branch (or if it is, the tracking configuration is missing or broken). "
1263 "Falling back to assuming the chromite checkout is derived from "
1264 "'master'; this *may* result in breakage." % cwd)
1268 def GarbageCollection(git_repo):
1269 """Cleanup unnecessary files and optimize the local repository.
1272 git_repo: Directory of git repository.
1274 # Use --auto so it only runs if housekeeping is necessary.
1275 RunGit(git_repo, ['gc', '--auto'])