2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Client-side script to send a try job to the try server. It communicates to
7 the try server by either writting to a svn/git repository or by directly
8 connecting to the server by HTTP.
29 import breakpad # pylint: disable=W0611
43 HELP_STRING = "Sorry, Tryserver is not available."
44 USAGE = r"""%prog [options]
46 Client-side script to send a try job to the try server. It communicates to
47 the try server by either writting to a svn repository or by directly connecting
48 to the server by HTTP."""
52 Send a patch directly from rietveld:
53 %(prog)s -R codereview.chromium.org/1337
54 --email recipient@example.com --root src
56 Try a change against a particular revision:
59 Try a change including changes to a sub repository:
60 %(prog)s -s third_party/WebKit
62 A git patch off a web site (git inserts a/ and b/) and fix the base dir:
63 %(prog)s --url http://url/to/patch.diff --patchlevel 1 --root src
65 Use svn to store the try job, specify an alternate email address and use a
66 premade diff file on the local drive:
67 %(prog)s --email user@example.com
68 --svn_repo svn://svn.chromium.org/chrome-try/try --diff foo.diff
70 Running only on a 'mac' slave with revision 123 and clobber first; specify
71 manually the 3 source files to use for the try job:
72 %(prog)s --bot mac --revision 123 --clobber -f src/a.cc -f src/a.h
76 GIT_PATCH_DIR_BASENAME = os.path.join('git-try', 'patches-git')
77 GIT_BRANCH_FILE = 'ref'
78 _GIT_PUSH_ATTEMPTS = 3
80 def DieWithError(message):
81 print >> sys.stderr, message
85 def RunCommand(args, error_ok=False, error_message=None, **kwargs):
87 return subprocess2.check_output(args, shell=False, **kwargs)
88 except subprocess2.CalledProcessError, e:
91 'Command "%s" failed.\n%s' % (
92 ' '.join(args), error_message or e.stdout or ''))
96 def RunGit(args, **kwargs):
98 return RunCommand(['git'] + args, **kwargs)
100 class Error(Exception):
101 """An error during a try job submission.
103 For this error, trychange.py does not display stack trace, only message
106 class InvalidScript(Error):
108 return self.args[0] + '\n' + HELP_STRING
111 class NoTryServerAccess(Error):
113 return self.args[0] + '\n' + HELP_STRING
116 """Escapes characters that could interfere with the file system or try job
119 return re.sub(r'[^\w#-]', '_', name)
123 """Simplistic base class to implement one function: ProcessOptions."""
124 def __init__(self, options, path, file_list):
125 items = path.split('@')
126 assert len(items) <= 2
127 self.checkout_root = os.path.abspath(items[0])
129 self.diff_against = items[1]
130 self.options = options
131 # Lazy-load file list from the SCM unless files were specified in options.
133 self._file_tuples = None
135 self._files = file_list
136 self._file_tuples = [('M', f) for f in self.files]
137 self.options.files = None
138 self.codereview_settings = None
139 self.codereview_settings_file = 'codereview.settings'
140 self.toplevel_root = None
142 def GetFileNames(self):
143 """Return the list of files in the diff."""
146 def GetCodeReviewSetting(self, key):
147 """Returns a value for the given key for this repository.
149 Uses gcl-style settings from the repository.
152 gcl_setting = gcl.GetCodeReviewSetting(key)
153 if gcl_setting != '':
155 if self.codereview_settings is None:
156 self.codereview_settings = {}
157 settings_file = self.ReadRootFile(self.codereview_settings_file)
159 for line in settings_file.splitlines():
160 if not line or line.lstrip().startswith('#'):
162 k, v = line.split(":", 1)
163 self.codereview_settings[k.strip()] = v.strip()
164 return self.codereview_settings.get(key, '')
166 def _GclStyleSettings(self):
167 """Set default settings based on the gcl-style settings from the repository.
169 The settings in the self.options object will only be set if no previous
170 value exists (i.e. command line flags to the try command will override the
171 settings in codereview.settings).
174 'port': self.GetCodeReviewSetting('TRYSERVER_HTTP_PORT'),
175 'host': self.GetCodeReviewSetting('TRYSERVER_HTTP_HOST'),
176 'svn_repo': self.GetCodeReviewSetting('TRYSERVER_SVN_URL'),
177 'gerrit_url': self.GetCodeReviewSetting('TRYSERVER_GERRIT_URL'),
178 'git_repo': self.GetCodeReviewSetting('TRYSERVER_GIT_URL'),
179 'project': self.GetCodeReviewSetting('TRYSERVER_PROJECT'),
180 # Primarily for revision=auto
181 'revision': self.GetCodeReviewSetting('TRYSERVER_REVISION'),
182 'root': self.GetCodeReviewSetting('TRYSERVER_ROOT'),
183 'patchlevel': self.GetCodeReviewSetting('TRYSERVER_PATCHLEVEL'),
185 logging.info('\n'.join(['%s: %s' % (k, v)
186 for (k, v) in settings.iteritems() if v]))
187 for (k, v) in settings.iteritems():
188 # Avoid overwriting options already set using command line flags.
189 if v and getattr(self.options, k) is None:
190 setattr(self.options, k, v)
192 def AutomagicalSettings(self):
193 """Determines settings based on supported code review and checkout tools.
195 # Try to find gclient or repo root first.
196 if not self.options.no_search:
197 self.toplevel_root = gclient_utils.FindGclientRoot(self.checkout_root)
198 if self.toplevel_root:
199 logging.info('Found .gclient at %s' % self.toplevel_root)
201 self.toplevel_root = gclient_utils.FindFileUpwards(
202 os.path.join('..', '.repo'), self.checkout_root)
203 if self.toplevel_root:
204 logging.info('Found .repo dir at %s'
205 % os.path.dirname(self.toplevel_root))
207 # Parse TRYSERVER_* settings from codereview.settings before falling back
208 # on setting self.options.root manually further down. Otherwise
209 # TRYSERVER_ROOT would never be used in codereview.settings.
210 self._GclStyleSettings()
212 if self.toplevel_root and not self.options.root:
213 assert os.path.abspath(self.toplevel_root) == self.toplevel_root
214 self.options.root = gclient_utils.PathDifference(self.toplevel_root,
217 self._GclStyleSettings()
219 def ReadRootFile(self, filename):
220 cur = self.checkout_root
221 root = self.toplevel_root or self.checkout_root
223 assert cur.startswith(root), (root, cur)
224 while cur.startswith(root):
225 filepath = os.path.join(cur, filename)
226 if os.path.isfile(filepath):
227 logging.info('Found %s at %s' % (filename, cur))
228 return gclient_utils.FileRead(filepath)
229 cur = os.path.dirname(cur)
230 logging.warning('Didn\'t find %s' % filename)
233 def _SetFileTuples(self, file_tuples):
234 excluded = ['!', '?', 'X', ' ', '~']
236 if f[0][0] in excluded:
238 for r in self.options.exclude:
239 if re.search(r, f[1]):
240 logging.info('Ignoring "%s"' % f[1])
244 self._file_tuples = [f for f in file_tuples if not Excluded(f)]
245 self._files = [f[1] for f in self._file_tuples]
247 def CaptureStatus(self):
248 """Returns the 'svn status' emulated output as an array of (status, file)
250 raise NotImplementedError(
251 "abstract method -- subclass %s must override" % self.__class__)
255 if self._files is None:
256 self._SetFileTuples(self.CaptureStatus())
260 def file_tuples(self):
261 if self._file_tuples is None:
262 self._SetFileTuples(self.CaptureStatus())
263 return self._file_tuples
267 """Gathers the options and diff for a subversion checkout."""
268 def __init__(self, *args, **kwargs):
269 SCM.__init__(self, *args, **kwargs)
270 self.checkout_root = scm.SVN.GetCheckoutRoot(self.checkout_root)
271 if not self.options.email:
272 # Assumes the svn credential is an email address.
273 self.options.email = scm.SVN.GetEmail(self.checkout_root)
274 logging.info("SVN(%s)" % self.checkout_root)
276 def ReadRootFile(self, filename):
277 data = SCM.ReadRootFile(self, filename)
281 # Try to search on the subversion repository for the file.
284 data = gcl.GetCachedFile(filename)
285 logging.debug('%s:\n%s' % (filename, data))
288 def CaptureStatus(self):
289 return scm.SVN.CaptureStatus(None, self.checkout_root)
291 def GenerateDiff(self):
292 """Returns a string containing the diff for the given file list.
294 The files in the list should either be absolute paths or relative to the
297 return scm.SVN.GenerateDiff(self.files, self.checkout_root, full_move=True,
298 revision=self.diff_against)
302 """Gathers the options and diff for a git checkout."""
303 def __init__(self, *args, **kwargs):
304 SCM.__init__(self, *args, **kwargs)
305 self.checkout_root = scm.GIT.GetCheckoutRoot(self.checkout_root)
306 if not self.options.name:
307 self.options.name = scm.GIT.GetPatchName(self.checkout_root)
308 if not self.options.email:
309 self.options.email = scm.GIT.GetEmail(self.checkout_root)
310 if not self.diff_against:
311 self.diff_against = scm.GIT.GetUpstreamBranch(self.checkout_root)
312 if not self.diff_against:
313 raise NoTryServerAccess(
314 "Unable to determine default branch to diff against. "
315 "Verify this branch is set up to track another"
316 "(via the --track argument to \"git checkout -b ...\"")
317 logging.info("GIT(%s)" % self.checkout_root)
319 def CaptureStatus(self):
320 return scm.GIT.CaptureStatus(
322 self.checkout_root.replace(os.sep, '/'),
325 def GenerateDiff(self):
326 if RunGit(['diff-index', 'HEAD']):
327 print 'Cannot try with a dirty tree. You must commit locally first.'
329 return scm.GIT.GenerateDiff(
333 branch=self.diff_against)
336 def _ParseBotList(botlist, testfilter):
337 """Parses bot configurations from a list of strings."""
340 for bot in itertools.chain.from_iterable(botspec.split(',')
341 for botspec in botlist):
344 if bot.endswith(':compile'):
345 tests |= set(['compile'])
348 'Can\'t use both --testfilter and --bot builder:test formats '
351 bots.append((bot, tests))
353 for botspec in botlist:
354 botname = botspec.split(':')[0]
357 tests |= set(filter(None, botspec.split(':')[1].split(',')))
358 bots.append((botname, tests))
362 def _ApplyTestFilter(testfilter, bot_spec):
363 """Applies testfilter from CLI.
365 Specifying a testfilter strips off any builder-specified tests (except for
369 return [(botname, set(testfilter) | (tests & set(['compile'])))
370 for botname, tests in bot_spec]
375 def _GenTSBotSpec(checkouts, change, changed_files, options):
377 # Get try slaves from PRESUBMIT.py files if not specified.
378 # Even if the diff comes from options.url, use the local checkout for bot
381 import presubmit_support
382 root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py')
384 if not changed_files:
385 changed_files = checkouts[0].file_tuples
386 change = presubmit_support.Change(options.name,
388 checkouts[0].checkout_root,
393 masters = presubmit_support.DoGetTryMasters(
395 checkouts[0].GetFileNames(),
396 checkouts[0].checkout_root,
402 # Compatibility for old checkouts and bots that were on tryserver.chromium.
403 trybots = masters.get('tryserver.chromium', [])
405 # Compatibility for checkouts that are not using tryserver.chromium
406 # but are stuck with git-try or gcl-try.
407 if not trybots and len(masters) == 1:
408 trybots = masters.values()[0]
411 old_style = filter(lambda x: isinstance(x, basestring), trybots)
412 new_style = filter(lambda x: isinstance(x, tuple), trybots)
414 # _ParseBotList's testfilter is set to None otherwise it will complain.
415 bot_spec = _ApplyTestFilter(options.testfilter,
416 _ParseBotList(old_style, None))
418 bot_spec.extend(_ApplyTestFilter(options.testfilter, new_style))
426 def _ParseSendChangeOptions(bot_spec, options):
427 """Parse common options passed to _SendChangeHTTP, _SendChangeSVN and
431 ('user', options.user),
432 ('name', options.name),
434 # A list of options to copy.
445 for option_name in optional_values:
446 value = getattr(options, option_name)
448 values.append((option_name, value))
450 # Not putting clobber to optional_names
451 # because it used to have lower-case 'true'.
453 values.append(('clobber', 'true'))
455 for bot, tests in bot_spec:
456 values.append(('bot', ('%s:%s' % (bot, ','.join(tests)))))
461 def _SendChangeHTTP(bot_spec, options):
462 """Send a change to the try server using the HTTP protocol."""
464 raise NoTryServerAccess('Please use the --host option to specify the try '
465 'server host to connect to.')
467 raise NoTryServerAccess('Please use the --port option to specify the try '
468 'server port to connect to.')
470 values = _ParseSendChangeOptions(bot_spec, options)
471 values.append(('patch', options.diff))
473 url = 'http://%s:%s/send_try_patch' % (options.host, options.port)
475 logging.info('Sending by HTTP')
476 logging.info(''.join("%s=%s\n" % (k, v) for k, v in values))
478 logging.info(options.diff)
483 logging.info('Opening connection...')
484 connection = urllib2.urlopen(url, urllib.urlencode(values))
488 if bot_spec and len(e.args) > 2 and e.args[2] == 'got a bad status line':
489 raise NoTryServerAccess('%s is unaccessible. Bad --bot argument?' % url)
491 raise NoTryServerAccess('%s is unaccessible. Reason: %s' % (url,
494 raise NoTryServerAccess('%s is unaccessible.' % url)
495 logging.info('Reading response...')
496 response = connection.read()
499 raise NoTryServerAccess('%s is unaccessible. Got:\n%s' % (url, response))
501 PrintSuccess(bot_spec, options)
503 @contextlib.contextmanager
504 def _TempFilename(name, contents=None):
505 """Create a temporary directory, append the specified name and yield.
507 In contrast to NamedTemporaryFile, does not keep the file open.
508 Deletes the file on __exit__.
510 temp_dir = tempfile.mkdtemp(prefix=name)
512 path = os.path.join(temp_dir, name)
513 if contents is not None:
514 with open(path, 'wb') as f:
518 shutil.rmtree(temp_dir, True)
521 @contextlib.contextmanager
522 def _PrepareDescriptionAndPatchFiles(description, options):
523 """Creates temporary files with description and patch.
525 __enter__ called on the return value returns a tuple of patch_filename and
526 description_filename.
529 description: contents of description file.
530 options: patchset options object. Must have attributes: user,
531 name (of patch) and diff (contents of patch).
533 current_time = str(datetime.datetime.now()).replace(':', '.')
534 patch_basename = '%s.%s.%s.diff' % (Escape(options.user),
535 Escape(options.name), current_time)
536 with _TempFilename('description', description) as description_filename:
537 with _TempFilename(patch_basename, options.diff) as patch_filename:
538 yield patch_filename, description_filename
541 def _SendChangeSVN(bot_spec, options):
542 """Send a change to the try server by committing a diff file on a subversion
544 if not options.svn_repo:
545 raise NoTryServerAccess('Please use the --svn_repo option to specify the'
546 ' try server svn repository to connect to.')
548 values = _ParseSendChangeOptions(bot_spec, options)
549 description = ''.join("%s=%s\n" % (k, v) for k, v in values)
550 logging.info('Sending by SVN')
551 logging.info(description)
552 logging.info(options.svn_repo)
553 logging.info(options.diff)
557 with _PrepareDescriptionAndPatchFiles(description, options) as (
558 patch_filename, description_filename):
559 if sys.platform == "cygwin":
560 # Small chromium-specific issue here:
561 # git-try uses /usr/bin/python on cygwin but svn.bat will be used
562 # instead of /usr/bin/svn by default. That causes bad things(tm) since
563 # Windows' svn.exe has no clue about cygwin paths. Hence force to use
564 # the cygwin version in this particular context.
568 patch_dir = os.path.dirname(patch_filename)
569 command = [exe, 'import', '-q', patch_dir, options.svn_repo, '--file',
570 description_filename]
571 if scm.SVN.AssertVersion("1.5")[0]:
572 command.append('--no-ignore')
575 subprocess2.check_call(command)
576 except subprocess2.CalledProcessError, e:
577 raise NoTryServerAccess(str(e))
579 PrintSuccess(bot_spec, options)
581 def _GetPatchGitRepo(git_url):
582 """Gets a path to a Git repo with patches.
584 Stores patches in .git/git-try/patches-git directory, a git repo. If it
585 doesn't exist yet or its origin URL is different, cleans up and clones it.
586 If it existed before, then pulls changes.
588 Does not support SVN repo.
590 Returns a path to the directory with patches.
592 git_dir = scm.GIT.GetGitDir(os.getcwd())
593 patch_dir = os.path.join(git_dir, GIT_PATCH_DIR_BASENAME)
595 logging.info('Looking for git repo for patches')
596 # Is there already a repo with the expected url or should we clone?
598 if os.path.exists(patch_dir) and scm.GIT.IsInsideWorkTree(patch_dir):
599 existing_url = scm.GIT.Capture(
600 ['config', '--local', 'remote.origin.url'],
602 clone = existing_url != git_url
605 if os.path.exists(patch_dir):
606 logging.info('Cleaning up')
607 shutil.rmtree(patch_dir, True)
608 logging.info('Cloning patch repo')
609 scm.GIT.Capture(['clone', git_url, GIT_PATCH_DIR_BASENAME], cwd=git_dir)
610 email = scm.GIT.GetEmail(cwd=os.getcwd())
611 scm.GIT.Capture(['config', '--local', 'user.email', email], cwd=patch_dir)
613 if scm.GIT.IsWorkTreeDirty(patch_dir):
614 logging.info('Work dir is dirty: hard reset!')
615 scm.GIT.Capture(['reset', '--hard'], cwd=patch_dir)
616 logging.info('Updating patch repo')
617 scm.GIT.Capture(['pull', 'origin', 'master'], cwd=patch_dir)
619 return os.path.abspath(patch_dir)
622 def _SendChangeGit(bot_spec, options):
623 """Sends a change to the try server by committing a diff file to a GIT repo.
625 Creates a temp orphan branch, commits patch.diff, creates a ref pointing to
626 that commit, deletes the temp branch, checks master out, adds 'ref' file
627 containing the name of the new ref, pushes master and the ref to the origin.
629 TODO: instead of creating a temp branch, use git-commit-tree.
632 if not options.git_repo:
633 raise NoTryServerAccess('Please use the --git_repo option to specify the '
634 'try server git repository to connect to.')
636 values = _ParseSendChangeOptions(bot_spec, options)
637 comment_subject = '%s.%s' % (options.user, options.name)
638 comment_body = ''.join("%s=%s\n" % (k, v) for k, v in values)
639 description = '%s\n\n%s' % (comment_subject, comment_body)
640 logging.info('Sending by GIT')
641 logging.info(description)
642 logging.info(options.git_repo)
643 logging.info(options.diff)
647 patch_dir = _GetPatchGitRepo(options.git_repo)
648 def patch_git(*args):
649 return scm.GIT.Capture(list(args), cwd=patch_dir)
650 def add_and_commit(filename, comment_filename):
651 patch_git('add', filename)
652 patch_git('commit', '-F', comment_filename)
654 assert scm.GIT.IsInsideWorkTree(patch_dir)
655 assert not scm.GIT.IsWorkTreeDirty(patch_dir)
657 with _PrepareDescriptionAndPatchFiles(description, options) as (
658 patch_filename, description_filename):
659 logging.info('Committing patch')
661 temp_branch = 'tmp_patch'
662 target_ref = 'refs/patches/%s/%s' % (
663 Escape(options.user),
664 os.path.basename(patch_filename).replace(' ','_'))
665 target_filename = os.path.join(patch_dir, 'patch.diff')
666 branch_file = os.path.join(patch_dir, GIT_BRANCH_FILE)
668 patch_git('checkout', 'master')
670 # Try deleting an existing temp branch, if any.
672 patch_git('branch', '-D', temp_branch)
673 logging.debug('Deleted an existing temp branch.')
674 except subprocess2.CalledProcessError:
676 # Create a new branch and put the patch there.
677 patch_git('checkout', '--orphan', temp_branch)
679 patch_git('clean', '-f')
680 shutil.copyfile(patch_filename, target_filename)
681 add_and_commit(target_filename, description_filename)
682 assert not scm.GIT.IsWorkTreeDirty(patch_dir)
684 # Create a ref and point it to the commit referenced by temp_branch.
685 patch_git('update-ref', target_ref, temp_branch)
687 # Delete the temp ref.
688 patch_git('checkout', 'master')
689 patch_git('branch', '-D', temp_branch)
691 # Update the branch file in the master.
693 with open(branch_file, 'w') as f:
695 add_and_commit(branch_file, description_filename)
699 # Push master and target_ref to origin.
700 logging.info('Pushing patch')
701 for attempt in xrange(_GIT_PUSH_ATTEMPTS):
703 patch_git('push', 'origin', 'master', target_ref)
704 except subprocess2.CalledProcessError as e:
705 is_last = attempt == _GIT_PUSH_ATTEMPTS - 1
707 raise NoTryServerAccess(str(e))
708 # Fetch, reset, update branch file again.
709 patch_git('fetch', 'origin')
710 patch_git('reset', '--hard', 'origin/master')
712 except subprocess2.CalledProcessError, e:
714 patch_git('checkout', 'master')
715 patch_git('reset', '--hard', 'origin/master')
718 PrintSuccess(bot_spec, options)
720 def _SendChangeGerrit(bot_spec, options):
721 """Posts a try job to a Gerrit change.
723 Reads Change-Id from the HEAD commit, resolves the current revision, checks
724 that local revision matches the uploaded one, posts a try job in form of a
725 message, sets Tryjob-Request label to 1.
727 Gerrit message format: starts with !tryjob, optionally followed by a tryjob
728 definition in JSON format:
729 buildNames: list of strings specifying build names.
732 logging.info('Sending by Gerrit')
733 if not options.gerrit_url:
734 raise NoTryServerAccess('Please use --gerrit_url option to specify the '
735 'Gerrit instance url to connect to')
736 gerrit_host = urlparse.urlparse(options.gerrit_url).hostname
737 logging.debug('Gerrit host: %s' % gerrit_host)
739 def GetChangeId(commmitish):
740 """Finds Change-ID of the HEAD commit."""
741 CHANGE_ID_RGX = '^Change-Id: (I[a-f0-9]{10,})'
742 comment = scm.GIT.Capture(['log', '-1', commmitish, '--format=%b'],
744 change_id_match = re.search(CHANGE_ID_RGX, comment, re.I | re.M)
745 if not change_id_match:
746 raise Error('Change-Id was not found in the HEAD commit. Make sure you '
747 'have a Git hook installed that generates and inserts a '
748 'Change-Id into a commit message automatically.')
749 change_id = change_id_match.group(1)
753 # Build job definition.
755 builderNames = [builder for builder, _ in bot_spec]
757 job_def['builderNames'] = builderNames
762 msg = '%s %s' % (msg, json.dumps(job_def, sort_keys=True))
765 def PostTryjob(message):
766 logging.info('Posting gerrit message: %s' % message)
767 if not options.dry_run:
768 # Post a message and set TryJob=1 label.
770 gerrit_util.SetReview(gerrit_host, change_id, msg=message,
771 labels={'Tryjob-Request': 1})
772 except gerrit_util.GerritError, e:
773 if e.http_status == 400:
774 raise Error(e.message)
778 head_sha = scm.GIT.Capture(['log', '-1', '--format=%H'], cwd=os.getcwd())
780 change_id = GetChangeId(head_sha)
783 # Check that the uploaded revision matches the local one.
784 changes = gerrit_util.GetChangeCurrentRevision(gerrit_host, change_id)
785 except gerrit_util.GerritAuthenticationError, e:
786 raise NoTryServerAccess(e.message)
788 assert len(changes) <= 1, 'Multiple changes with id %s' % change_id
790 raise Error('A change %s was not found on the server. Was it uploaded?' %
792 logging.debug('Found Gerrit change: %s' % changes[0])
793 if changes[0]['current_revision'] != head_sha:
794 raise Error('Please upload your latest local changes to Gerrit.')
797 message = FormatMessage()
799 change_url = urlparse.urljoin(options.gerrit_url,
800 '/#/c/%s' % changes[0]['_number'])
801 print('A tryjob was posted on change %s' % change_url)
803 def PrintSuccess(bot_spec, options):
804 if not options.dry_run:
805 text = 'Patch \'%s\' sent to try server' % options.name
807 text += ': %s' % ', '.join(
808 '%s:%s' % (b[0], ','.join(b[1])) for b in bot_spec)
812 def GuessVCS(options, path, file_list):
813 """Helper to guess the version control system.
815 NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't
818 This examines the path directory, guesses which SCM we're using, and
819 returns an instance of the appropriate class. Exit with an error if we can't
823 A SCM instance. Exits if the SCM can't be guessed.
825 __pychecker__ = 'no-returnvalues'
826 real_path = path.split('@')[0]
827 logging.info("GuessVCS(%s)" % path)
828 # Subversion has a .svn in all working directories.
829 if os.path.isdir(os.path.join(real_path, '.svn')):
830 return SVN(options, path, file_list)
832 # Git has a command to test if you're in a git tree.
833 # Try running it, but don't die if we don't have git installed.
835 subprocess2.check_output(
836 ['git', 'rev-parse', '--is-inside-work-tree'], cwd=real_path,
837 stderr=subprocess2.VOID)
838 return GIT(options, path, file_list)
840 if e.errno != errno.ENOENT:
842 except subprocess2.CalledProcessError, e:
843 if e.returncode != errno.ENOENT and e.returncode != 128:
844 # ENOENT == 2 = they don't have git installed.
845 # 128 = git error code when not in a repo.
846 logging.warning('Unexpected error code: %s' % e.returncode)
848 raise NoTryServerAccess(
849 ( 'Could not guess version control system for %s.\n'
850 'Are you in a working copy directory?') % path)
853 def GetMungedDiff(path_diff, diff):
854 # Munge paths to match svn.
856 for i in range(len(diff)):
857 if diff[i].startswith('--- ') or diff[i].startswith('+++ '):
858 new_file = posixpath.join(path_diff, diff[i][4:]).replace('\\', '/')
859 if diff[i].startswith('--- '):
860 file_path = new_file.split('\t')[0].strip()
861 if file_path.startswith('a/'):
862 file_path = file_path[2:]
863 changed_files.append(('M', file_path))
864 diff[i] = diff[i][0:4] + new_file
865 return (diff, changed_files)
868 class OptionParser(optparse.OptionParser):
869 def format_epilog(self, _):
870 """Removes epilog formatting."""
871 return self.epilog or ''
874 def gen_parser(prog):
876 parser = OptionParser(usage=USAGE, version=__version__, prog=prog)
877 parser.add_option("-v", "--verbose", action="count", default=0,
878 help="Prints debugging infos")
879 group = optparse.OptionGroup(parser, "Result and status")
880 group.add_option("-u", "--user", default=getpass.getuser(),
881 help="Owner user name [default: %default]")
882 group.add_option("-e", "--email",
883 default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
884 os.environ.get('EMAIL_ADDRESS')),
885 help="Email address where to send the results. Use either "
886 "the TRYBOT_RESULTS_EMAIL_ADDRESS environment "
887 "variable or EMAIL_ADDRESS to set the email address "
888 "the try bots report results to [default: %default]")
889 group.add_option("-n", "--name",
890 help="Descriptive name of the try job")
891 group.add_option("--issue", type='int',
892 help="Update rietveld issue try job status")
893 group.add_option("--patchset", type='int',
894 help="Update rietveld issue try job status. This is "
895 "optional if --issue is used, In that case, the "
896 "latest patchset will be used.")
897 group.add_option("--dry_run", action='store_true',
898 help="Don't send the try job. This implies --verbose, so "
899 "it will print the diff.")
900 parser.add_option_group(group)
902 group = optparse.OptionGroup(parser, "Try job options")
904 "-b", "--bot", action="append",
905 help=("IMPORTANT: specify ONE builder per --bot flag. Use it multiple "
906 "times to specify multiple builders. ex: "
907 "'-bwin_rel:ui_tests,webkit_unit_tests -bwin_layout'. See "
908 "the try server waterfall for the builders name and the tests "
909 "available. Can also be used to specify gtest_filter, e.g. "
910 "-bwin_rel:base_unittests:ValuesTest.*Value"))
911 group.add_option("-B", "--print_bots", action="store_true",
912 help="Print bots we would use (e.g. from PRESUBMIT.py)"
913 " and exit. Do not send patch. Like --dry_run"
914 " but less verbose.")
915 group.add_option("-r", "--revision",
916 help="Revision to use for the try job. If 'auto' is "
917 "specified, it is resolved to the revision a patch is "
918 "generated against (Git only). Default: the "
919 "revision will be determined by the try server; see "
920 "its waterfall for more info")
921 group.add_option("-c", "--clobber", action="store_true",
922 help="Force a clobber before building; e.g. don't do an "
924 # TODO(maruel): help="Select a specific configuration, usually 'debug' or "
926 group.add_option("--target", help=optparse.SUPPRESS_HELP)
928 group.add_option("--project",
929 help="Override which project to use. Projects are defined "
930 "server-side to define what default bot set to use")
933 "-t", "--testfilter", action="append", default=[],
934 help=("Apply a testfilter to all the selected builders. Unless the "
935 "builders configurations are similar, use multiple "
936 "--bot <builder>:<test> arguments."))
938 parser.add_option_group(group)
940 group = optparse.OptionGroup(parser, "Patch to run")
941 group.add_option("-f", "--file", default=[], dest="files",
942 metavar="FILE", action="append",
943 help="Use many times to list the files to include in the "
944 "try, relative to the repository root")
945 group.add_option("--diff",
946 help="File containing the diff to try")
947 group.add_option("--url",
948 help="Url where to grab a patch, e.g. "
949 "http://example.com/x.diff")
950 group.add_option("-R", "--rietveld_url", default="codereview.appspot.com",
952 help="Has 2 usages, both refer to the rietveld instance: "
953 "Specify which code review patch to use as the try job "
954 "or rietveld instance to update the try job results "
956 group.add_option("--root",
957 help="Root to use for the patch; base subdirectory for "
958 "patch created in a subdirectory")
959 group.add_option("-p", "--patchlevel", type='int', metavar="LEVEL",
960 help="Used as -pN parameter to patch")
961 group.add_option("-s", "--sub_rep", action="append", default=[],
962 help="Subcheckout to use in addition. This is mainly "
963 "useful for gclient-style checkouts. In git, checkout "
964 "the branch with changes first. Use @rev or "
965 "@branch to specify the "
966 "revision/branch to diff against. If no @branch is "
967 "given the diff will be against the upstream branch. "
968 "If @branch then the diff is branch..HEAD. "
969 "All edits must be checked in.")
970 group.add_option("--no_search", action="store_true",
971 help=("Disable automatic search for gclient or repo "
973 group.add_option("-E", "--exclude", action="append",
974 default=['ChangeLog'], metavar='REGEXP',
975 help="Regexp patterns to exclude files. Default: %default")
976 group.add_option("--upstream_branch", action="store",
977 help="Specify the upstream branch to diff against in the "
979 parser.add_option_group(group)
981 group = optparse.OptionGroup(parser, "Access the try server by HTTP")
982 group.add_option("--use_http",
983 action="store_const",
984 const=_SendChangeHTTP,
986 help="Use HTTP to talk to the try server [default]")
987 group.add_option("-H", "--host",
989 group.add_option("-P", "--port", type="int",
991 parser.add_option_group(group)
993 group = optparse.OptionGroup(parser, "Access the try server with SVN")
994 group.add_option("--use_svn",
995 action="store_const",
996 const=_SendChangeSVN,
998 help="Use SVN to talk to the try server")
999 group.add_option("-S", "--svn_repo",
1001 help="SVN url to use to write the changes in; --use_svn is "
1002 "implied when using --svn_repo")
1003 parser.add_option_group(group)
1005 group = optparse.OptionGroup(parser, "Access the try server with Git")
1006 group.add_option("--use_git",
1007 action="store_const",
1008 const=_SendChangeGit,
1010 help="Use GIT to talk to the try server")
1011 group.add_option("-G", "--git_repo",
1013 help="GIT url to use to write the changes in; --use_git is "
1014 "implied when using --git_repo")
1015 parser.add_option_group(group)
1017 group = optparse.OptionGroup(parser, "Access the try server with Gerrit")
1018 group.add_option("--use_gerrit",
1019 action="store_const",
1020 const=_SendChangeGerrit,
1022 help="Use Gerrit to talk to the try server")
1023 group.add_option("--gerrit_url",
1024 metavar="GERRIT_URL",
1025 help="Gerrit url to post a tryjob to; --use_gerrit is "
1026 "implied when using --gerrit_url")
1027 parser.add_option_group(group)
1039 argv: Arguments and options.
1040 change: Change instance corresponding to the CL.
1041 swallow_exception: Whether we raise or swallow exceptions.
1043 parser = gen_parser(prog)
1044 epilog = EPILOG % { 'prog': prog }
1046 epilog += extra_epilog
1047 parser.epilog = epilog
1049 options, args = parser.parse_args(argv)
1051 # If they've asked for help, give it to them
1052 if len(args) == 1 and args[0] == 'help':
1056 # If they've said something confusing, don't spawn a try job until you
1057 # understand what they want.
1059 parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
1062 options.verbose += 1
1064 LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s'
1065 if not swallow_exception:
1066 if options.verbose == 0:
1067 logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT)
1068 elif options.verbose == 1:
1069 logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
1070 elif options.verbose > 1:
1071 logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
1075 if (options.patchlevel is not None and
1076 (options.patchlevel < 0 or options.patchlevel > 10)):
1078 'Have you tried --port instead? You probably confused -p and -P.')
1080 # Strip off any @ in the user, otherwise svn gets confused.
1081 options.user = options.user.split('@', 1)[0]
1083 if options.rietveld_url:
1084 # Try to extract the review number if possible and fix the protocol.
1085 if not '://' in options.rietveld_url:
1086 options.rietveld_url = 'http://' + options.rietveld_url
1087 match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url)
1089 if options.issue or options.patchset:
1090 parser.error('Cannot use both --issue and use a review number url')
1091 options.issue = int(match.group(2))
1092 options.rietveld_url = match.group(1)
1095 changed_files = None
1096 # Always include os.getcwd() in the checkout settings.
1101 file_list = options.files
1103 file_list = [f.LocalPath() for f in change.AffectedFiles()]
1105 if options.upstream_branch:
1106 path += '@' + options.upstream_branch
1107 # Clear file list so that the correct list will be retrieved from the
1111 current_vcs = GuessVCS(options, path, file_list)
1112 current_vcs.AutomagicalSettings()
1113 options = current_vcs.options
1114 vcs_is_git = type(current_vcs) is GIT
1116 # So far, git_repo doesn't work with SVN
1117 if options.git_repo and not vcs_is_git:
1118 parser.error('--git_repo option is supported only for GIT repositories')
1120 # If revision==auto, resolve it
1121 if options.revision and options.revision.lower() == 'auto':
1123 parser.error('--revision=auto is supported only for GIT repositories')
1124 options.revision = scm.GIT.Capture(
1125 ['rev-parse', current_vcs.diff_against],
1128 checkouts = [current_vcs]
1129 for item in options.sub_rep:
1130 # Pass file_list=None because we don't know the sub repo's file list.
1131 checkout = GuessVCS(options,
1132 os.path.join(current_vcs.checkout_root, item),
1134 if checkout.checkout_root in [c.checkout_root for c in checkouts]:
1135 parser.error('Specified the root %s two times.' %
1136 checkout.checkout_root)
1137 checkouts.append(checkout)
1139 can_http = options.port and options.host
1140 can_svn = options.svn_repo
1141 can_git = options.git_repo
1142 can_gerrit = options.gerrit_url
1143 can_something = can_http or can_svn or can_git or can_gerrit
1144 # If there was no transport selected yet, now we must have enough data to
1146 if not options.send_patch and not can_something:
1147 parser.error('Please specify an access method.')
1149 # Convert options.diff into the content of the diff.
1152 parser.error('You cannot specify files and --url at the same time.')
1153 options.diff = urllib2.urlopen(options.url).read()
1156 parser.error('You cannot specify files and --diff at the same time.')
1157 options.diff = gclient_utils.FileRead(options.diff, 'rb')
1158 elif options.issue and options.patchset is None:
1159 # Retrieve the patch from rietveld when the diff is not specified.
1160 # When patchset is specified, it's because it's done by gcl/git-try.
1161 api_url = '%s/api/%d' % (options.rietveld_url, options.issue)
1162 logging.debug(api_url)
1163 contents = json.loads(urllib2.urlopen(api_url).read())
1164 options.patchset = contents['patchsets'][-1]
1165 diff_url = ('%s/download/issue%d_%d.diff' %
1166 (options.rietveld_url, options.issue, options.patchset))
1167 diff = GetMungedDiff('', urllib2.urlopen(diff_url).readlines())
1168 options.diff = ''.join(diff[0])
1169 changed_files = diff[1]
1171 # Use this as the base.
1172 root = checkouts[0].checkout_root
1174 for checkout in checkouts:
1175 raw_diff = checkout.GenerateDiff()
1178 diff = raw_diff.splitlines(True)
1179 path_diff = gclient_utils.PathDifference(root, checkout.checkout_root)
1181 diffs.extend(GetMungedDiff(path_diff, diff)[0])
1183 logging.error('Empty or non-existant diff, exiting.')
1185 options.diff = ''.join(diffs)
1187 if not options.name:
1189 options.name = 'Issue %s' % options.issue
1191 options.name = 'Unnamed'
1192 print('Note: use --name NAME to change the try job name.')
1194 if not options.email:
1195 parser.error('Using an anonymous checkout. Please use --email or set '
1196 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.')
1197 print('Results will be emailed to: ' + options.email)
1200 bot_spec = _ApplyTestFilter(
1201 options.testfilter, _ParseBotList(options.bot, options.testfilter))
1203 bot_spec = _GenTSBotSpec(checkouts, change, changed_files, options)
1205 if options.testfilter:
1206 bot_spec = _ApplyTestFilter(options.testfilter, bot_spec)
1208 if any('triggered' in b[0] for b in bot_spec):
1209 print >> sys.stderr, (
1210 'ERROR You are trying to send a job to a triggered bot. This type of'
1211 ' bot requires an\ninitial job from a parent (usually a builder). '
1212 'Instead send your job to the parent.\nBot list: %s' % bot_spec)
1215 if options.print_bots:
1216 print 'Bots which would be used:'
1217 for bot in bot_spec:
1219 print ' %s:%s' % (bot[0], ','.join(bot[1]))
1221 print ' %s' % (bot[0])
1224 # Determine sending protocol
1225 if options.send_patch:
1227 senders = [options.send_patch]
1229 # Try sending patch using avaialble protocols
1231 (_SendChangeHTTP, can_http),
1232 (_SendChangeSVN, can_svn),
1233 (_SendChangeGerrit, can_gerrit),
1234 (_SendChangeGit, can_git),
1236 senders = [sender for sender, can in all_senders if can]
1239 for sender in senders:
1241 sender(bot_spec, options)
1243 except NoTryServerAccess:
1244 is_last = sender == senders[-1]
1247 assert False, "Unreachable code"
1249 if swallow_exception:
1251 print >> sys.stderr, e
1253 except (gclient_utils.Error, subprocess2.CalledProcessError), e:
1254 print >> sys.stderr, e
1259 if __name__ == "__main__":
1260 fix_encoding.fix_encoding()
1261 sys.exit(TryChange(None, None, False))