Tools/Scripts/validate-committer-lists

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2009, Google Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30 #
  31 # Checks Python's known list of committers against lists.webkit.org and SVN history.
  32
  33
  34 import os
  35 import subprocess
  36 import re
  37 import urllib2
  38 from datetime import date, datetime, timedelta
  39 from optparse import OptionParser
  40
  41 from webkitpy.common.config.committers import CommitterList
  42 from webkitpy.common.system.deprecated_logging import log, error
  43 from webkitpy.common.checkout.scm import Git
  44 from webkitpy.common.net.bugzilla import Bugzilla
  45
  46 # WebKit includes a built copy of BeautifulSoup in Scripts/webkitpy
  47 # so this import should always succeed.
  48 from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup
  49
  50
  51 def print_list_if_non_empty(title, list_to_print):
  52     if not list_to_print:
  53         return
  54     print # Newline before the list
  55     print title
  56     for item in list_to_print:
  57         print item
  58
  59
  60 class CommitterListFromMailingList(object):
  61     committers_list_url = "http://lists.webkit.org/mailman/roster.cgi/webkit-committers"
  62     reviewers_list_url = "http://lists.webkit.org/mailman/roster.cgi/webkit-reviewers"
  63
  64     def _fetch_emails_from_page(self, url):
  65         page = urllib2.urlopen(url)
  66         soup = BeautifulSoup(page)
  67
  68         emails = []
  69         # Grab the cells in the first column (which happens to be the bug ids).
  70         for email_item in soup('li'):
  71             email_link = email_item.find("a")
  72             email = email_link.string.replace(" at ", "@") # The email is obfuscated using " at " instead of "@".
  73             emails.append(email)
  74         return emails
  75
  76     @staticmethod
  77     def _commiters_not_found_in_email_list(committers, emails):
  78         missing_from_mailing_list = []
  79         for committer in committers:
  80             for email in committer.emails:
  81                 if email in emails:
  82                     break
  83             else:
  84                 missing_from_mailing_list.append(committer)
  85         return missing_from_mailing_list
  86
  87     @staticmethod
  88     def _emails_not_found_in_committer_list(committers, emails):
  89         email_to_committer_map = {}
  90         for committer in committers:
  91             for email in committer.emails:
  92                 email_to_committer_map[email] = committer
  93
  94         return filter(lambda email: not email_to_committer_map.get(email), emails)
  95
  96     def check_for_emails_missing_from_list(self, committer_list):
  97         committer_emails = self._fetch_emails_from_page(self.committers_list_url)
  98         list_name = "webkit-committers@lists.webkit.org"
  99
 100         missing_from_mailing_list = self._commiters_not_found_in_email_list(committer_list.committers(), committer_emails)
 101         print_list_if_non_empty("Committers missing from %s:" % list_name, missing_from_mailing_list)
 102
 103         users_missing_from_committers = self._emails_not_found_in_committer_list(committer_list.committers(), committer_emails)
 104         print_list_if_non_empty("Subcribers to %s missing from committer.py:" % list_name, users_missing_from_committers)
 105
 106
 107         reviewer_emails = self._fetch_emails_from_page(self.reviewers_list_url)
 108         list_name = "webkit-reviewers@lists.webkit.org"
 109
 110         missing_from_mailing_list = self._commiters_not_found_in_email_list(committer_list.reviewers(), reviewer_emails)
 111         print_list_if_non_empty("Reviewers missing from %s:" % list_name, missing_from_mailing_list)
 112
 113         missing_from_reviewers = self._emails_not_found_in_committer_list(committer_list.reviewers(), reviewer_emails)
 114         print_list_if_non_empty("Subcribers to %s missing from reviewers in committer.py:" % list_name, missing_from_reviewers)
 115
 116         missing_from_committers = self._emails_not_found_in_committer_list(committer_list.committers(), reviewer_emails)
 117         print_list_if_non_empty("Subcribers to %s completely missing from committers.py" % list_name, missing_from_committers)
 118
 119
 120 class CommitterListFromGit(object):
 121     login_to_email_address = {
 122         'aliceli1' : 'alice.liu@apple.com',
 123         'bdash' : 'mrowe@apple.com',
 124         'bdibello' : 'bdibello@apple.com', # Bruce DiBello, only 4 commits: r10023, r9548, r9538, r9535
 125         'cblu' : 'cblu@apple.com',
 126         'cpeterse' : 'cpetersen@apple.com',
 127         'eseidel' : 'eric@webkit.org',
 128         'gdennis' : 'gdennis@webkit.org',
 129         'goldsmit' : 'goldsmit@apple.com', # Debbie Goldsmith, only one commit r8839
 130         'gramps' : 'gramps@apple.com',
 131         'honeycutt' : 'jhoneycutt@apple.com',
 132         'jdevalk' : 'joost@webkit.org',
 133         'jens' : 'jens@apple.com',
 134         'justing' : 'justin.garcia@apple.com',
 135         'kali' : 'kali@apple.com', # Christy Warren, did BIDI work, 5 commits: r8815, r8802, r8801, r8791, r8773, r8603
 136         'kjk' : 'kkowalczyk@gmail.com',
 137         'kmccullo' : 'kmccullough@apple.com',
 138         'kocienda' : 'kocienda@apple.com',
 139         'lamadio' : 'lamadio@apple.com', # Lou Amadio, only 2 commits: r17949 and r17783
 140         'lars' : 'lars@kde.org',
 141         'lweintraub' : 'lweintraub@apple.com',
 142         'lypanov' : 'lypanov@kde.org',
 143         'mhay' : 'mhay@apple.com', # Mike Hay, 3 commits: r3813, r2552, r2548
 144         'ouch' : 'ouch@apple.com', # John Louch
 145         'pyeh' : 'patti@apple.com', # Patti Yeh, did VoiceOver work in WebKit
 146         'rjw' : 'rjw@apple.com',
 147         'seangies' : 'seangies@apple.com', # Sean Gies?, only 5 commits: r16600, r16592, r16511, r16489, r16484
 148         'sheridan' : 'sheridan@apple.com', # Shelly Sheridan
 149         'thatcher' : 'timothy@apple.com',
 150         'tomernic' : 'timo@apple.com',
 151         'trey' : 'trey@usa.net',
 152         'tristan' : 'tristan@apple.com',
 153         'vicki' : 'vicki@apple.com',
 154         'voas' : 'voas@apple.com', # Ed Voas, did some Carbon work in WebKit
 155         'zack' : 'zack@kde.org',
 156         'zimmermann' : 'zimmermann@webkit.org',
 157     }
 158
 159     def __init__(self):
 160         self._last_commit_time_by_author_cache = {}
 161
 162     def _fetch_authors_and_last_commit_time_from_git_log(self):
 163         last_commit_dates = {}
 164         git_log_args = ['git', 'log', '--reverse', '--pretty=format:%ae %at']
 165         process = subprocess.Popen(git_log_args, stdout=subprocess.PIPE)
 166
 167         # eric@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc 1257090899
 168         line_regexp = re.compile("^(?P<author>.+)@\S+ (?P<timestamp>\d+)$")
 169         while True:
 170             output_line = process.stdout.readline()
 171             if output_line == '' and process.poll() != None:
 172                 return last_commit_dates
 173
 174             match_result = line_regexp.match(output_line)
 175             if not match_result:
 176                 error("Failed to match line: %s" % output_line)
 177             last_commit_dates[match_result.group('author')] = float(match_result.group('timestamp'))
 178
 179     def _fill_in_emails_for_old_logins(self):
 180         authors_missing_email = filter(lambda author: author.find('@') == -1, self._last_commit_time_by_author_cache)
 181         authors_with_email = filter(lambda author: author.find('@') != -1, self._last_commit_time_by_author_cache)
 182         prefixes_of_authors_with_email = map(lambda author: author.split('@')[0], authors_with_email)
 183
 184         for author in authors_missing_email:
 185             # First check to see if we have a manual mapping from login to email.
 186             author_email = self.login_to_email_address.get(author)
 187
 188             # Most old logins like 'darin' are now just 'darin@apple.com', so check for a prefix match if a manual mapping was not found.
 189             if not author_email and author in prefixes_of_authors_with_email:
 190                 author_email_index = prefixes_of_authors_with_email.index(author)
 191                 author_email = authors_with_email[author_email_index]
 192
 193             if not author_email:
 194                 # No known email mapping, likely not an active committer.  We could log here.
 195                 continue
 196
 197             # log("%s -> %s" % (author, author_email)) # For sanity checking.
 198             no_email_commit_time = self._last_commit_time_by_author_cache.get(author)
 199             email_commit_time = self._last_commit_time_by_author_cache.get(author_email)
 200             # We compare the timestamps for extra sanity even though we could assume commits before email address were used for login are always going to be older.
 201             if not email_commit_time or email_commit_time < no_email_commit_time:
 202                 self._last_commit_time_by_author_cache[author_email] = no_email_commit_time
 203             del self._last_commit_time_by_author_cache[author]
 204
 205     def _last_commit_by_author(self):
 206         if not self._last_commit_time_by_author_cache:
 207             self._last_commit_time_by_author_cache = self._fetch_authors_and_last_commit_time_from_git_log()
 208             self._fill_in_emails_for_old_logins()
 209             del self._last_commit_time_by_author_cache['(no author)'] # The initial svn import isn't very useful.
 210         return self._last_commit_time_by_author_cache
 211
 212     @staticmethod
 213     def _print_three_column_row(widths, values):
 214         print "%s%s%s" % (values[0].ljust(widths[0]), values[1].ljust(widths[1]), values[2])
 215
 216     def print_possibly_expired_committers(self, committer_list):
 217         authors_and_last_commits = self._last_commit_by_author().items()
 218         authors_and_last_commits.sort(lambda a,b: cmp(a[1], b[1]), reverse=True)
 219         committer_cuttof = date.today() - timedelta(days=365)
 220         column_widths = [13, 25]
 221         print
 222         print "Committers who have not committed within one year:"
 223         self._print_three_column_row(column_widths, ("Last Commit", "Committer Email", "Committer Record"))
 224         for (author, last_commit) in authors_and_last_commits:
 225             last_commit_date = date.fromtimestamp(last_commit)
 226             if committer_cuttof > last_commit_date:
 227                 committer_record = committer_list.committer_by_email(author)
 228                 self._print_three_column_row(column_widths, (str(last_commit_date), author, committer_record))
 229
 230     def print_committers_missing_from_committer_list(self, committer_list):
 231         missing_from_committers_py = []
 232         last_commit_time_by_author = self._last_commit_by_author()
 233         for author in last_commit_time_by_author:
 234             if not committer_list.committer_by_email(author):
 235                 missing_from_committers_py.append(author)
 236
 237         never_committed = []
 238         for committer in committer_list.committers():
 239             for email in committer.emails:
 240                 if last_commit_time_by_author.get(email):
 241                     break
 242             else:
 243                 never_committed.append(committer)
 244
 245         print_list_if_non_empty("Historical committers missing from committer.py:", missing_from_committers_py)
 246         print_list_if_non_empty("Committers in committer.py who have never committed:", never_committed)
 247
 248
 249 class CommitterListBugzillaChecker(object):
 250     def __init__(self):
 251         self._bugzilla = Bugzilla()
 252
 253     def _has_invalid_bugzilla_email(self, committer):
 254         return not self._bugzilla.queries.fetch_logins_matching_substring(committer.bugzilla_email())
 255
 256     def print_committers_with_invalid_bugzilla_emails(self, committer_list):
 257         print # Print a newline before we start hitting bugzilla (it logs about logging in).
 258         print "Checking committer emails against bugzilla (this will take a long time)"
 259         committers_with_invalid_bugzilla_email = filter(self._has_invalid_bugzilla_email, committer_list.committers())
 260         print_list_if_non_empty("Committers with invalid bugzilla email:", committers_with_invalid_bugzilla_email)
 261
 262
 263 def main():
 264     parser = OptionParser()
 265     parser.add_option("-b", "--check-bugzilla-emails", action="store_true", help="Check the bugzilla_email for each committer against bugs.webkit.org")
 266     (options, args) = parser.parse_args()
 267
 268     committer_list = CommitterList()
 269     CommitterListFromMailingList().check_for_emails_missing_from_list(committer_list)
 270
 271     if not Git.in_working_directory("."):
 272         print """\n\nWARNING: validate-committer-lists requires a git checkout.
 273 The following checks are disabled:
 274  - List of committers ordered by last commit
 275  - List of historical committers missing from committers.py
 276 """
 277         return 1
 278     svn_committer_list = CommitterListFromGit()
 279     svn_committer_list.print_possibly_expired_committers(committer_list)
 280     svn_committer_list.print_committers_missing_from_committer_list(committer_list)
 281
 282     if options.check_bugzilla_emails:
 283         CommitterListBugzillaChecker().print_committers_with_invalid_bugzilla_emails(committer_list)
 284
 285
 286 if __name__ == "__main__":
 287     main()