1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
6 """Top-level presubmit script for Skia.
8 See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
9 for more details about the presubmit API built into gcl.
22 REVERT_CL_SUBJECT_PREFIX = 'Revert '
24 SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
26 # Please add the complete email address here (and not just 'xyz@' or 'xyz').
30 'bsalomon@chromium.org',
31 'bsalomon@google.com',
32 'djsollen@chromium.org',
33 'djsollen@google.com',
36 AUTHORS_FILE_NAME = 'AUTHORS'
38 DOCS_PREVIEW_URL = 'https://skia.org/?cl='
39 GOLD_TRYBOT_URL = ('https://gold.skia.org/search2?unt=true'
40 '&query=source_type%3Dgm&master=false&issue=')
42 # Path to CQ bots feature is described in https://bug.skia.org/4364
43 PATH_PREFIX_TO_EXTRA_TRYBOTS = {
44 # pylint: disable=line-too-long
45 'cmake/': 'client.skia.compile:Build-Mac10.9-Clang-x86_64-Release-CMake-Trybot,Build-Ubuntu-GCC-x86_64-Release-CMake-Trybot',
46 # pylint: disable=line-too-long
47 'src/opts/': 'client.skia:Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD-Trybot',
49 'include/private/SkAtomics.h': ('client.skia:'
50 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-TSAN-Trybot,'
51 'Test-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-TSAN-Trybot'
54 # Below are examples to show what is possible with this feature.
55 # 'src/svg/': 'master1:abc;master2:def',
56 # 'src/svg/parser/': 'master3:ghi,jkl;master4:mno',
57 # 'src/image/SkImage_Base.h': 'master5:pqr,stu;master1:abc1;master2:def',
61 def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
62 """Checks that files end with atleast one \n (LF)."""
64 for f in input_api.AffectedSourceFiles(source_file_filter):
65 contents = input_api.ReadFile(f, 'rb')
66 # Check that the file ends in atleast one newline character.
67 if len(contents) > 1 and contents[-1:] != '\n':
68 eof_files.append(f.LocalPath())
71 return [output_api.PresubmitPromptWarning(
72 'These files should end in a newline character:',
77 def _PythonChecks(input_api, output_api):
78 """Run checks on any modified Python files."""
79 pylint_disabled_warnings = (
80 'F0401', # Unable to import.
81 'E0611', # No name in module.
82 'W0232', # Class has no __init__ method.
83 'E1002', # Use of super on an old style class.
84 'W0403', # Relative import used.
85 'R0201', # Method could be a function.
86 'E1003', # Using class name in super.
87 'W0613', # Unused argument.
88 'W0105', # String statement has no effect.
90 # Run Pylint on only the modified python files. Unfortunately it still runs
91 # Pylint on the whole file instead of just the modified lines.
92 affected_python_files = []
93 for affected_file in input_api.AffectedSourceFiles(None):
94 affected_file_path = affected_file.LocalPath()
95 if affected_file_path.endswith('.py'):
96 affected_python_files.append(affected_file_path)
97 return input_api.canned_checks.RunPylint(
98 input_api, output_api,
99 disabled_warnings=pylint_disabled_warnings,
100 white_list=affected_python_files)
103 def _IfDefChecks(input_api, output_api):
104 """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
105 comment_block_start_pattern = re.compile('^\s*\/\*.*$')
106 comment_block_middle_pattern = re.compile('^\s+\*.*')
107 comment_block_end_pattern = re.compile('^\s+\*\/.*$')
108 single_line_comment_pattern = re.compile('^\s*//.*$')
109 def is_comment(line):
110 return (comment_block_start_pattern.match(line) or
111 comment_block_middle_pattern.match(line) or
112 comment_block_end_pattern.match(line) or
113 single_line_comment_pattern.match(line))
115 empty_line_pattern = re.compile('^\s*$')
116 def is_empty_line(line):
117 return empty_line_pattern.match(line)
120 for affected_file in input_api.AffectedSourceFiles(None):
121 affected_file_path = affected_file.LocalPath()
122 if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
123 f = open(affected_file_path)
124 for line in f.xreadlines():
125 if is_comment(line) or is_empty_line(line):
127 # The below will be the first real line after comments and newlines.
128 if line.startswith('#if 0 '):
130 elif line.startswith('#if ') or line.startswith('#ifdef '):
131 failing_files.append(affected_file_path)
137 output_api.PresubmitError(
138 'The following files have #if or #ifdef before includes:\n%s\n\n'
139 'See https://bug.skia.org/3362 for why this should be fixed.' %
140 '\n'.join(failing_files)))
144 def _CopyrightChecks(input_api, output_api, source_file_filter=None):
146 year_pattern = r'\d{4}'
147 year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
148 years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
149 copyright_pattern = (
150 r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
152 for affected_file in input_api.AffectedSourceFiles(source_file_filter):
153 if 'third_party' in affected_file.LocalPath():
155 contents = input_api.ReadFile(affected_file, 'rb')
156 if not re.search(copyright_pattern, contents):
157 results.append(output_api.PresubmitError(
158 '%s is missing a correct copyright header.' % affected_file))
162 def _ToolFlags(input_api, output_api):
163 """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
165 sources = lambda x: ('dm_flags.py' in x.LocalPath() or
166 'nanobench_flags.py' in x.LocalPath())
167 for f in input_api.AffectedSourceFiles(sources):
168 if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
169 results.append(output_api.PresubmitError('`python %s test` failed' % f))
173 def _CommonChecks(input_api, output_api):
174 """Presubmit checks common to upload and commit."""
176 sources = lambda x: (x.LocalPath().endswith('.h') or
177 x.LocalPath().endswith('.gypi') or
178 x.LocalPath().endswith('.gyp') or
179 x.LocalPath().endswith('.py') or
180 x.LocalPath().endswith('.sh') or
181 x.LocalPath().endswith('.m') or
182 x.LocalPath().endswith('.mm') or
183 x.LocalPath().endswith('.go') or
184 x.LocalPath().endswith('.c') or
185 x.LocalPath().endswith('.cc') or
186 x.LocalPath().endswith('.cpp'))
189 input_api, output_api, source_file_filter=sources))
190 results.extend(_PythonChecks(input_api, output_api))
191 results.extend(_IfDefChecks(input_api, output_api))
192 results.extend(_CopyrightChecks(input_api, output_api,
193 source_file_filter=sources))
194 results.extend(_ToolFlags(input_api, output_api))
198 def CheckChangeOnUpload(input_api, output_api):
199 """Presubmit checks for the change on upload.
201 The following are the presubmit checks:
202 * Check change has one and only one EOL.
205 results.extend(_CommonChecks(input_api, output_api))
209 def _CheckTreeStatus(input_api, output_api, json_url):
210 """Check whether to allow commit.
213 input_api: input related apis.
214 output_api: output related apis.
215 json_url: url to download json style status.
217 tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
218 input_api, output_api, json_url=json_url)
219 if not tree_status_results:
220 # Check for caution state only if tree is not closed.
221 connection = input_api.urllib2.urlopen(json_url)
222 status = input_api.json.loads(connection.read())
224 if ('caution' in status['message'].lower() and
225 os.isatty(sys.stdout.fileno())):
226 # Display a prompt only if we are in an interactive shell. Without this
227 # check the commit queue behaves incorrectly because it considers
228 # prompts to be failures.
229 short_text = 'Tree state is: ' + status['general_state']
230 long_text = status['message'] + '\n' + json_url
231 tree_status_results.append(
232 output_api.PresubmitPromptWarning(
233 message=short_text, long_text=long_text))
235 # Tree status is closed. Put in message about contacting sheriff.
236 connection = input_api.urllib2.urlopen(
237 SKIA_TREE_STATUS_URL + '/current-sheriff')
238 sheriff_details = input_api.json.loads(connection.read())
240 tree_status_results[0]._message += (
241 '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
242 'to submit a build fix\nand do not know how to submit because the '
243 'tree is closed') % sheriff_details['username']
244 return tree_status_results
247 def _CheckOwnerIsInAuthorsFile(input_api, output_api):
249 issue = input_api.change.issue
250 if issue and input_api.rietveld:
251 issue_properties = input_api.rietveld.get_issue_properties(
252 issue=int(issue), messages=False)
253 owner_email = issue_properties['owner_email']
257 for line in open(AUTHORS_FILE_NAME):
258 if not line.startswith('#'):
259 authors_content += line
260 email_fnmatches = re.findall('<(.*)>', authors_content)
261 for email_fnmatch in email_fnmatches:
262 if fnmatch.fnmatch(owner_email, email_fnmatch):
263 # Found a match, the user is in the AUTHORS file break out of the loop
267 output_api.PresubmitError(
268 'The email %s is not in Skia\'s AUTHORS file.\n'
269 'Issue owner, this CL must include an addition to the Skia AUTHORS '
273 # Do not fail if authors file cannot be found.
274 traceback.print_exc()
275 input_api.logging.error('AUTHORS file not found!')
280 def _CheckLGTMsForPublicAPI(input_api, output_api):
281 """Check LGTMs for public API changes.
283 For public API files make sure there is an LGTM from the list of owners in
287 requires_owner_check = False
288 for affected_file in input_api.AffectedFiles():
289 affected_file_path = affected_file.LocalPath()
290 file_path, file_ext = os.path.splitext(affected_file_path)
291 # We only care about files that end in .h and are under the top-level
292 # include dir, but not include/private.
293 if (file_ext == '.h' and
294 'include' == file_path.split(os.path.sep)[0] and
295 'private' not in file_path):
296 requires_owner_check = True
298 if not requires_owner_check:
301 lgtm_from_owner = False
302 issue = input_api.change.issue
303 if issue and input_api.rietveld:
304 issue_properties = input_api.rietveld.get_issue_properties(
305 issue=int(issue), messages=True)
306 if re.match(REVERT_CL_SUBJECT_PREFIX, issue_properties['subject'], re.I):
307 # It is a revert CL, ignore the public api owners check.
310 if issue_properties['cq_dry_run']:
311 # Ignore public api owners check for dry run CLs since they are not
312 # going to be committed.
315 match = re.search(r'^TBR=(.*)$', issue_properties['description'], re.M)
317 tbr_entries = match.group(1).strip().split(',')
318 for owner in PUBLIC_API_OWNERS:
319 if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
320 # If an owner is specified in the TBR= line then ignore the public
324 if issue_properties['owner_email'] in PUBLIC_API_OWNERS:
325 # An owner created the CL that is an automatic LGTM.
326 lgtm_from_owner = True
328 messages = issue_properties.get('messages')
330 for message in messages:
331 if (message['sender'] in PUBLIC_API_OWNERS and
332 'lgtm' in message['text'].lower()):
333 # Found an lgtm in a message from an owner.
334 lgtm_from_owner = True
337 if not lgtm_from_owner:
339 output_api.PresubmitError(
340 "If this CL adds to or changes Skia's public API, you need an LGTM "
341 "from any of %s. If this CL only removes from or doesn't change "
342 "Skia's public API, please add a short note to the CL saying so "
343 "and add one of those reviewers on a TBR= line. If you don't know "
344 "if this CL affects Skia's public API, treat it like it does."
345 % str(PUBLIC_API_OWNERS)))
349 def PostUploadHook(cl, change, output_api):
350 """git cl upload will call this hook after the issue is created/modified.
352 This hook does the following:
353 * Adds a link to the CL's Gold trybot results.
354 * Adds a link to preview docs changes if there are any docs changes in the CL.
355 * Adds 'NOTRY=true' if the CL contains only docs changes.
356 * Adds 'NOTREECHECKS=true' for non master branch changes since they do not
357 need to be gated on the master branch's tree.
358 * Adds 'NOTRY=true' for non master branch changes since trybots do not yet
360 * Adds 'NOPRESUBMIT=true' for non master branch changes since those don't
361 run the presubmit checks.
362 * Adds extra trybots for the paths defined in PATH_TO_EXTRA_TRYBOTS.
366 atleast_one_docs_change = False
367 all_docs_changes = True
368 for affected_file in change.AffectedFiles():
369 affected_file_path = affected_file.LocalPath()
370 file_path, _ = os.path.splitext(affected_file_path)
371 if 'site' == file_path.split(os.path.sep)[0]:
372 atleast_one_docs_change = True
374 all_docs_changes = False
375 if atleast_one_docs_change and not all_docs_changes:
379 rietveld_obj = cl.RpcServer()
380 if issue and rietveld_obj:
381 original_description = rietveld_obj.get_description(issue)
382 new_description = original_description
384 # Add GOLD_TRYBOT_URL if it does not exist yet.
385 if not re.search(r'^GOLD_TRYBOT_URL=', new_description, re.M | re.I):
386 new_description += '\nGOLD_TRYBOT_URL= %s%s' % (GOLD_TRYBOT_URL, issue)
388 output_api.PresubmitNotifyResult(
389 'Added link to Gold trybot runs to the CL\'s description.\n'
390 'Note: Results may take sometime to be populated after trybots '
393 # If the change includes only doc changes then add NOTRY=true in the
394 # CL's description if it does not exist yet.
395 if all_docs_changes and not re.search(
396 r'^NOTRY=true$', new_description, re.M | re.I):
397 new_description += '\nNOTRY=true'
399 output_api.PresubmitNotifyResult(
400 'This change has only doc changes. Automatically added '
401 '\'NOTRY=true\' to the CL\'s description'))
403 # If there is atleast one docs change then add preview link in the CL's
404 # description if it does not already exist there.
405 if atleast_one_docs_change and not re.search(
406 r'^DOCS_PREVIEW=.*', new_description, re.M | re.I):
407 # Automatically add a link to where the docs can be previewed.
408 new_description += '\nDOCS_PREVIEW= %s%s' % (DOCS_PREVIEW_URL, issue)
410 output_api.PresubmitNotifyResult(
411 'Automatically added a link to preview the docs changes to the '
412 'CL\'s description'))
414 # If the target ref is not master then add NOTREECHECKS=true and NOTRY=true
415 # to the CL's description if it does not already exist there.
416 target_ref = rietveld_obj.get_issue_properties(issue, False).get(
418 if target_ref != 'refs/heads/master':
420 r'^NOTREECHECKS=true$', new_description, re.M | re.I):
421 new_description += "\nNOTREECHECKS=true"
423 output_api.PresubmitNotifyResult(
424 'Branch changes do not need to rely on the master branch\'s '
425 'tree status. Automatically added \'NOTREECHECKS=true\' to the '
426 'CL\'s description'))
428 r'^NOTRY=true$', new_description, re.M | re.I):
429 new_description += "\nNOTRY=true"
431 output_api.PresubmitNotifyResult(
432 'Trybots do not yet work for non-master branches. '
433 'Automatically added \'NOTRY=true\' to the CL\'s description'))
435 r'^NOPRESUBMIT=true$', new_description, re.M | re.I):
436 new_description += "\nNOPRESUBMIT=true"
438 output_api.PresubmitNotifyResult(
439 'Branch changes do not run the presubmit checks.'))
441 # Automatically set CQ_EXTRA_TRYBOTS if any of the changed files here begin
442 # with the paths of interest.
443 cq_master_to_trybots = collections.defaultdict(set)
444 for affected_file in change.AffectedFiles():
445 affected_file_path = affected_file.LocalPath()
446 for path_prefix, extra_bots in PATH_PREFIX_TO_EXTRA_TRYBOTS.iteritems():
447 if affected_file_path.startswith(path_prefix):
449 output_api.PresubmitNotifyResult(
450 'Your CL modifies the path %s.\nAutomatically adding %s to '
451 'the CL description.' % (affected_file_path, extra_bots)))
452 _MergeCQExtraTrybotsMaps(
453 cq_master_to_trybots, _GetCQExtraTrybotsMap(extra_bots))
454 if cq_master_to_trybots:
455 new_description = _AddCQExtraTrybotsToDesc(
456 cq_master_to_trybots, new_description)
458 # If the description has changed update it.
459 if new_description != original_description:
460 rietveld_obj.update_description(issue, new_description)
465 def _AddCQExtraTrybotsToDesc(cq_master_to_trybots, description):
466 """Adds the specified master and trybots to the CQ_EXTRA_TRYBOTS keyword.
468 If the keyword already exists in the description then it appends to it only
469 if the specified values do not already exist.
470 If the keyword does not exist then it creates a new section in the
473 match = re.search(r'^CQ_EXTRA_TRYBOTS=(.*)$', description, re.M | re.I)
475 original_trybots_map = _GetCQExtraTrybotsMap(match.group(1))
476 _MergeCQExtraTrybotsMaps(cq_master_to_trybots, original_trybots_map)
477 new_description = description.replace(
478 match.group(0), _GetCQExtraTrybotsStr(cq_master_to_trybots))
480 new_description = description + "\n%s" % (
481 _GetCQExtraTrybotsStr(cq_master_to_trybots))
482 return new_description
485 def _MergeCQExtraTrybotsMaps(dest_map, map_to_be_consumed):
486 """Merges two maps of masters to trybots into one."""
487 for master, trybots in map_to_be_consumed.iteritems():
488 dest_map[master].update(trybots)
492 def _GetCQExtraTrybotsMap(cq_extra_trybots_str):
493 """Parses the CQ_EXTRA_TRYBOTS str and returns a map of masters to trybots."""
494 cq_master_to_trybots = collections.defaultdict(set)
495 for section in cq_extra_trybots_str.split(';'):
497 master, bots = section.split(':')
498 cq_master_to_trybots[master].update(bots.split(','))
499 return cq_master_to_trybots
502 def _GetCQExtraTrybotsStr(cq_master_to_trybots):
503 """Constructs the CQ_EXTRA_TRYBOTS str from a map of masters to trybots."""
505 for master, trybots in cq_master_to_trybots.iteritems():
506 sections.append('%s:%s' % (master, ','.join(trybots)))
507 return 'CQ_EXTRA_TRYBOTS=%s' % ';'.join(sections)
510 def CheckChangeOnCommit(input_api, output_api):
511 """Presubmit checks for the change on commit.
513 The following are the presubmit checks:
514 * Check change has one and only one EOL.
515 * Ensures that the Skia tree is open in
516 http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
517 state and an error if it is in 'Closed' state.
520 results.extend(_CommonChecks(input_api, output_api))
522 _CheckTreeStatus(input_api, output_api, json_url=(
523 SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
524 results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
525 results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))