2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Makes sure files have the right permissions.
8 Some developers have broken SCM configurations that flip the svn:executable
9 permission on for no good reason. Unix developers who run ls --color will then
10 see .cc files in green and get confused.
12 - For file extensions that must be executable, add it to EXECUTABLE_EXTENSIONS.
13 - For file extensions that must not be executable, add it to
14 NOT_EXECUTABLE_EXTENSIONS.
15 - To ignore all the files inside a directory, add it to IGNORED_PATHS.
16 - For file base name with ambiguous state and that should not be checked for
17 shebang, add it to IGNORED_FILENAMES.
19 Any file not matching the above will be opened and looked if it has a shebang.
20 It this doesn't match the executable bit on the file, the file will be flagged.
22 Note that all directory separators must be slashes (Unix-style) and not
23 backslashes. All directories should be relative to the source root and all
24 file paths should be only lowercase.
34 #### USER EDITABLE SECTION STARTS HERE ####
36 # Files with these extensions must have executable bit set.
37 EXECUTABLE_EXTENSIONS = (
44 # These files must have executable bit set.
46 # TODO(maruel): Detect ELF files.
47 'chrome/installer/mac/sign_app.sh.in',
48 'chrome/installer/mac/sign_versioned_dir.sh.in',
51 # These files must not have the executable bit set. This is mainly a performance
52 # optimization as these files are not checked for shebang. The list was
53 # partially generated from:
54 # git ls-files | grep "\\." | sed 's/.*\.//' | sort | uniq -c | sort -b -g
55 NON_EXECUTABLE_EXTENSIONS = (
125 # File names that are always whitelisted. (These are all autoconf spew.)
126 IGNORED_FILENAMES = (
138 # File paths starting with one of these will be ignored as well.
139 # Please consider fixing your file permissions, rather than adding to this list.
141 # TODO(maruel): Detect ELF files.
142 'chrome/test/data/extensions/uitest/plugins/plugin.plugin/contents/'
143 'macos/testnetscapeplugin',
144 'chrome/test/data/extensions/uitest/plugins_private/plugin.plugin/contents/'
145 'macos/testnetscapeplugin',
146 'chrome/installer/mac/sign_app.sh.in',
147 'chrome/installer/mac/sign_versioned_dir.sh.in',
148 'native_client_sdk/src/build_tools/sdk_tools/third_party/',
150 # TODO(maruel): Fix these.
151 'third_party/android_testrunner/',
152 'third_party/bintrees/',
153 'third_party/closure_linter/',
154 'third_party/devscripts/licensecheck.pl.vanilla',
155 'third_party/hyphen/',
156 'third_party/jemalloc/',
157 'third_party/lcov-1.9/contrib/galaxy/conglomerate_functions.pl',
158 'third_party/lcov-1.9/contrib/galaxy/gen_makefile.sh',
159 'third_party/lcov/contrib/galaxy/conglomerate_functions.pl',
160 'third_party/lcov/contrib/galaxy/gen_makefile.sh',
161 'third_party/libevent/autogen.sh',
162 'third_party/libevent/test/test.sh',
163 'third_party/libxml/linux/xml2-config',
164 'third_party/libxml/src/ltmain.sh',
166 'third_party/protobuf/',
167 'third_party/python_gflags/gflags.py',
168 'third_party/sqlite/',
169 'third_party/talloc/script/mksyms.sh',
170 'third_party/tcmalloc/',
171 'third_party/tlslite/setup.py',
174 #### USER EDITABLE SECTION ENDS HERE ####
176 assert set(EXECUTABLE_EXTENSIONS) & set(NON_EXECUTABLE_EXTENSIONS) == set()
179 def capture(cmd, cwd):
180 """Returns the output of a command.
182 Ignores the error code or stderr.
184 logging.debug('%s; cwd=%s' % (' '.join(cmd), cwd))
185 env = os.environ.copy()
186 env['LANGUAGE'] = 'en_US.UTF-8'
187 p = subprocess.Popen(
188 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, env=env)
189 return p.communicate()[0]
192 def get_svn_info(dir_path):
193 """Returns svn meta-data for a svn checkout."""
194 if not os.path.isdir(dir_path):
196 out = capture(['svn', 'info', '.', '--non-interactive'], dir_path)
197 return dict(l.split(': ', 1) for l in out.splitlines() if l)
200 def get_svn_url(dir_path):
201 return get_svn_info(dir_path).get('URL')
204 def get_svn_root(dir_path):
205 """Returns the svn checkout root or None."""
206 svn_url = get_svn_url(dir_path)
209 logging.info('svn url: %s' % svn_url)
211 parent = os.path.dirname(dir_path)
212 if parent == dir_path:
214 svn_url = svn_url.rsplit('/', 1)[0]
215 if svn_url != get_svn_url(parent):
220 def get_git_root(dir_path):
221 """Returns the git checkout root or None."""
222 root = capture(['git', 'rev-parse', '--show-toplevel'], dir_path).strip()
227 def is_ignored(rel_path):
228 """Returns True if rel_path is in our whitelist of files to ignore."""
229 rel_path = rel_path.lower()
231 os.path.basename(rel_path) in IGNORED_FILENAMES or
232 rel_path.startswith(IGNORED_PATHS))
235 def must_be_executable(rel_path):
236 """The file name represents a file type that must have the executable bit
240 os.path.splitext(rel_path)[1][1:].lower() in EXECUTABLE_EXTENSIONS or
241 rel_path in EXECUTABLE_PATHS)
244 def must_not_be_executable(rel_path):
245 """The file name represents a file type that must not have the executable
248 return os.path.splitext(rel_path)[1][1:].lower() in NON_EXECUTABLE_EXTENSIONS
251 def has_executable_bit(full_path):
252 """Returns if any executable bit is set."""
253 permission = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
254 return bool(permission & os.stat(full_path).st_mode)
257 def has_shebang(full_path):
258 """Returns if the file starts with #!/.
260 file_path is the absolute path to the file.
262 with open(full_path, 'rb') as f:
263 return f.read(3) == '#!/'
265 def check_file(full_path, bare_output):
266 """Checks file_path's permissions and returns an error if it is
269 It is assumed that the file is not ignored by is_ignored().
271 If the file name is matched with must_be_executable() or
272 must_not_be_executable(), only its executable bit is checked.
273 Otherwise, the 3 first bytes of the file are read to verify if it has a
274 shebang and compares this with the executable bit on the file.
277 bit = has_executable_bit(full_path)
279 # It's faster to catch exception than call os.path.islink(). Chromium
280 # tree happens to have invalid symlinks under
281 # third_party/openssl/openssl/test/.
284 if must_be_executable(full_path):
288 return '%s: Must have executable bit set' % full_path
290 if must_not_be_executable(full_path):
294 return '%s: Must not have executable bit set' % full_path
297 # For the others, it depends on the shebang.
298 shebang = has_shebang(full_path)
303 return '%s: Has executable bit but not shebang' % full_path
305 return '%s: Has shebang but not executable bit' % full_path
308 def check_files(root, files, bare_output):
310 for file_path in files:
311 if is_ignored(file_path):
314 full_file_path = os.path.join(root, file_path)
316 error = check_file(full_file_path, bare_output)
322 class ApiBase(object):
323 def __init__(self, root_dir, bare_output):
324 self.root_dir = root_dir
325 self.bare_output = bare_output
327 self.count_shebang = 0
329 def check_file(self, rel_path):
330 logging.debug('check_file(%s)' % rel_path)
333 if (not must_be_executable(rel_path) and
334 not must_not_be_executable(rel_path)):
335 self.count_shebang += 1
337 full_path = os.path.join(self.root_dir, rel_path)
338 return check_file(full_path, self.bare_output)
340 def check_dir(self, rel_path):
341 return self.check(rel_path)
343 def check(self, start_dir):
344 """Check the files in start_dir, recursively check its subdirectories."""
346 items = self.list_dir(start_dir)
347 logging.info('check(%s) -> %d' % (start_dir, len(items)))
349 full_path = os.path.join(self.root_dir, start_dir, item)
350 rel_path = full_path[len(self.root_dir) + 1:]
351 if is_ignored(rel_path):
353 if os.path.isdir(full_path):
355 errors.extend(self.check_dir(rel_path))
357 error = self.check_file(rel_path)
362 def list_dir(self, start_dir):
363 """Lists all the files and directory inside start_dir."""
365 x for x in os.listdir(os.path.join(self.root_dir, start_dir))
366 if not x.startswith('.')
370 class ApiSvnQuick(ApiBase):
371 """Returns all files in svn-versioned directories, independent of the fact if
374 Uses svn info in each directory to determine which directories should be
377 def __init__(self, *args):
378 super(ApiSvnQuick, self).__init__(*args)
379 self.url = get_svn_url(self.root_dir)
381 def check_dir(self, rel_path):
382 url = self.url + '/' + rel_path
383 if get_svn_url(os.path.join(self.root_dir, rel_path)) != url:
385 return super(ApiSvnQuick, self).check_dir(rel_path)
388 class ApiAllFilesAtOnceBase(ApiBase):
391 def list_dir(self, start_dir):
392 """Lists all the files and directory inside start_dir."""
393 if self._files is None:
394 self._files = sorted(self._get_all_files())
395 if not self.bare_output:
396 print 'Found %s files' % len(self._files)
397 start_dir = start_dir[len(self.root_dir) + 1:]
399 x[len(start_dir):] for x in self._files if x.startswith(start_dir)
402 def _get_all_files(self):
403 """Lists all the files and directory inside self._root_dir."""
404 raise NotImplementedError()
407 class ApiSvn(ApiAllFilesAtOnceBase):
408 """Returns all the subversion controlled files.
410 Warning: svn ls is abnormally slow.
412 def _get_all_files(self):
413 cmd = ['svn', 'ls', '--non-interactive', '--recursive']
415 x for x in capture(cmd, self.root_dir).splitlines()
416 if not x.endswith(os.path.sep))
419 class ApiGit(ApiAllFilesAtOnceBase):
420 def _get_all_files(self):
421 return capture(['git', 'ls-files'], cwd=self.root_dir).splitlines()
424 def get_scm(dir_path, bare):
425 """Returns a properly configured ApiBase instance."""
427 root = get_svn_root(dir_path or cwd)
430 print('Found subversion checkout at %s' % root)
431 return ApiSvnQuick(dir_path or root, bare)
432 root = get_git_root(dir_path or cwd)
435 print('Found git repository at %s' % root)
436 return ApiGit(dir_path or root, bare)
438 # Returns a non-scm aware checker.
440 print('Failed to determine the SCM for %s' % dir_path)
441 return ApiBase(dir_path or cwd, bare)
445 usage = """Usage: python %prog [--root <root>] [tocheck]
446 tocheck Specifies the directory, relative to root, to check. This defaults
447 to "." so it checks everything.
451 python %prog --root /path/to/source chrome"""
453 parser = optparse.OptionParser(usage=usage)
456 help='Specifies the repository root. This defaults '
457 'to the checkout repository root')
459 '-v', '--verbose', action='count', default=0, help='Print debug logging')
464 help='Prints the bare filename triggering the checks')
466 '--file', action='append', dest='files',
467 help='Specifics a list of files to check the permissions of. Only these '
468 'files will be checked')
469 options, args = parser.parse_args()
471 levels = [logging.ERROR, logging.INFO, logging.DEBUG]
472 logging.basicConfig(level=levels[min(len(levels) - 1, options.verbose)])
475 parser.error('Too many arguments used')
478 options.root = os.path.abspath(options.root)
481 errors = check_files(options.root, options.files, options.bare)
482 print '\n'.join(errors)
485 api = get_scm(options.root, options.bare)
489 start_dir = api.root_dir
491 errors = api.check(start_dir)
494 print 'Processed %s files, %d files where tested for shebang' % (
495 api.count, api.count_shebang)
500 print '\n'.join(errors)
507 if '__main__' == __name__: