2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
6 """Checks third-party licenses for the purposes of the Android WebView build.
8 The Android tree includes a snapshot of Chromium in order to power the system
9 WebView. This tool checks that all code uses open-source licenses compatible
10 with Android, and that we meet the requirements of those licenses. It can also
11 be used to generate an Android NOTICE file for the third-party code.
13 It makes use of src/tools/licenses.py and the README.chromium files on which
14 it depends. It also makes use of a data file, third_party_files_whitelist.txt,
15 which whitelists indicidual files which contain third-party code but which
16 aren't in a third-party directory with a README.chromium file.
29 REPOSITORY_ROOT = os.path.abspath(os.path.join(
30 os.path.dirname(__file__), '..', '..'))
32 # Import third_party/PRESUBMIT.py via imp to avoid importing a random
33 # PRESUBMIT.py from $PATH, also make sure we don't generate a .pyc file.
34 sys.dont_write_bytecode = True
36 imp.load_source('PRESUBMIT', \
37 os.path.join(REPOSITORY_ROOT, 'third_party', 'PRESUBMIT.py'))
39 sys.path.append(os.path.join(REPOSITORY_ROOT, 'tools'))
44 class InputApi(object):
48 def GetIncompatibleDirectories():
49 """Gets a list of third-party directories which use licenses incompatible
50 with Android. This is used by the snapshot tool.
52 A list of directories.
56 for directory in _FindThirdPartyDirs():
57 if directory in known_issues.KNOWN_ISSUES:
58 result.append(directory)
61 metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
62 require_license_file=False)
63 except licenses.LicenseError as e:
64 print 'Got LicenseError while scanning ' + directory
66 if metadata.get('License Android Compatible', 'no').upper() == 'YES':
68 license = re.split(' [Ll]icenses?$', metadata['License'])[0]
69 if not third_party.LicenseIsCompatibleWithAndroid(InputApi(), license):
70 result.append(directory)
73 def GetUnknownIncompatibleDirectories():
74 """Gets a list of third-party directories which use licenses incompatible
75 with Android which are not present in the known_issues.py file.
76 This is used by the AOSP bot.
78 A list of directories.
80 incompatible_directories = frozenset(GetIncompatibleDirectories())
81 known_incompatible = []
82 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
83 for exclude in exclude_list:
84 if glob.has_magic(exclude):
85 exclude_dirname = os.path.dirname(exclude)
86 if glob.has_magic(exclude_dirname):
87 print ('Exclude path %s contains an unexpected glob expression,' \
88 ' skipping.' % exclude)
89 exclude = exclude_dirname
90 known_incompatible.append(os.path.normpath(os.path.join(path, exclude)))
91 known_incompatible = frozenset(known_incompatible)
92 return incompatible_directories.difference(known_incompatible)
95 class ScanResult(object):
96 Ok, Warnings, Errors = range(3)
98 def _CheckLicenseHeaders(excluded_dirs_list, whitelisted_files):
99 """Checks that all files which are not in a listed third-party directory,
100 and which do not use the standard Chromium license, are whitelisted.
102 excluded_dirs_list: The list of directories to exclude from scanning.
103 whitelisted_files: The whitelist of files.
105 ScanResult.Ok if all files with non-standard license headers are whitelisted
106 and the whitelist contains no stale entries;
107 ScanResult.Warnings if there are stale entries;
108 ScanResult.Errors if new non-whitelisted entries found.
111 excluded_dirs_list = [d for d in excluded_dirs_list if not 'third_party' in d]
112 # Using a common pattern for third-partyies makes the ignore regexp shorter
113 excluded_dirs_list.append('third_party')
115 excluded_dirs_list.append('.git')
116 excluded_dirs_list.append('.svn')
118 excluded_dirs_list.append('out/Debug')
119 excluded_dirs_list.append('out/Release')
120 # 'Copyright' appears in license agreements
121 excluded_dirs_list.append('chrome/app/resources')
122 # Quickoffice js files from internal src used on buildbots. crbug.com/350472.
123 excluded_dirs_list.append('chrome/browser/resources/chromeos/quickoffice')
124 # This is a test output directory
125 excluded_dirs_list.append('chrome/tools/test/reference_build')
126 # blink style copy right headers.
127 excluded_dirs_list.append('content/shell/renderer/test_runner')
128 # blink style copy right headers.
129 excluded_dirs_list.append('content/shell/tools/plugin')
130 # This is tests directory, doesn't exist in the snapshot
131 excluded_dirs_list.append('content/test/data')
132 # This is a tests directory that doesn't exist in the shipped product.
133 excluded_dirs_list.append('gin/test')
134 # This is a test output directory
135 excluded_dirs_list.append('data/dom_perf')
136 # This is a tests directory that doesn't exist in the shipped product.
137 excluded_dirs_list.append('tools/perf/page_sets')
138 excluded_dirs_list.append('tools/perf/page_sets/tough_animation_cases')
139 # Histogram tools, doesn't exist in the snapshot
140 excluded_dirs_list.append('tools/histograms')
141 # Swarming tools, doesn't exist in the snapshot
142 excluded_dirs_list.append('tools/swarming_client')
143 # Arm sysroot tools, doesn't exist in the snapshot
144 excluded_dirs_list.append('arm-sysroot')
145 # Data is not part of open source chromium, but are included on some bots.
146 excluded_dirs_list.append('data')
147 # This is not part of open source chromium, but are included on some bots.
148 excluded_dirs_list.append('skia/tools/clusterfuzz-data')
150 args = ['android_webview/tools/find_copyrights.pl',
152 ] + excluded_dirs_list
153 p = subprocess.Popen(args=args, cwd=REPOSITORY_ROOT, stdout=subprocess.PIPE)
154 lines = p.communicate()[0].splitlines()
157 allowed_copyrights = '^(?:\*No copyright\*' \
158 '|20[0-9][0-9](?:-20[0-9][0-9])? The Chromium Authors\. ' \
159 'All rights reserved.*)$'
160 allowed_copyrights_re = re.compile(allowed_copyrights)
162 entries = l.split('\t')
163 if entries[1] == "GENERATED FILE":
165 copyrights = entries[1].split(' / ')
167 if c and not allowed_copyrights_re.match(c):
168 offending_files.append(os.path.normpath(entries[0]))
171 unknown = set(offending_files) - set(whitelisted_files)
173 print 'The following files contain a third-party license but are not in ' \
174 'a listed third-party directory and are not whitelisted. You must ' \
175 'add the following files to the whitelist.\n%s' % \
176 '\n'.join(sorted(unknown))
178 stale = set(whitelisted_files) - set(offending_files)
180 print 'The following files are whitelisted unnecessarily. You must ' \
181 'remove the following files from the whitelist.\n%s' % \
182 '\n'.join(sorted(stale))
185 return ScanResult.Errors
187 return ScanResult.Warnings
193 """Reads a file from disk.
195 path: The path of the file to read, relative to the root of the repository.
197 The contents of the file as a string.
200 return open(os.path.join(REPOSITORY_ROOT, path), 'rb').read()
203 def _FindThirdPartyDirs():
204 """Gets the list of third-party directories.
206 The list of third-party directories.
209 # Please don't add here paths that have problems with license files,
210 # as they will end up included in Android WebView snapshot.
211 # Instead, add them into known_issues.py.
213 # Temporary until we figure out how not to check out quickoffice on the
214 # Android license check bot. Tracked in crbug.com/350472.
215 os.path.join('chrome', 'browser', 'resources', 'chromeos', 'quickoffice'),
216 # Placeholder directory, no third-party code.
217 os.path.join('third_party', 'adobe'),
218 # Apache 2.0 license. See
219 # https://code.google.com/p/chromium/issues/detail?id=140478.
220 os.path.join('third_party', 'bidichecker'),
221 # Isn't checked out on clients
222 os.path.join('third_party', 'gles2_conform'),
223 # The llvm-build doesn't exist for non-clang builder
224 os.path.join('third_party', 'llvm-build'),
225 # Binaries doesn't apply to android
226 os.path.join('third_party', 'widevine'),
227 # third_party directories in this tree aren't actually third party, but
228 # provide a way to shadow experimental buildfiles into those directories.
229 os.path.join('tools', 'gn', 'secondary'),
230 # Not shipped, Chromium code
231 os.path.join('tools', 'swarming_client'),
233 third_party_dirs = licenses.FindThirdPartyDirs(prune_paths, REPOSITORY_ROOT)
234 return licenses.FilterDirsWithFiles(third_party_dirs, REPOSITORY_ROOT)
238 """Checks that license meta-data is present for all third-party code and
239 that all non third-party code doesn't contain external copyrighted code.
241 ScanResult.Ok if everything is in order;
242 ScanResult.Warnings if there are non-fatal problems (e.g. stale whitelist
244 ScanResult.Errors otherwise.
247 third_party_dirs = _FindThirdPartyDirs()
249 # First, check designated third-party directories using src/tools/licenses.py.
250 all_licenses_valid = True
251 for path in sorted(third_party_dirs):
253 licenses.ParseDir(path, REPOSITORY_ROOT)
254 except licenses.LicenseError, e:
255 if not (path in known_issues.KNOWN_ISSUES):
256 print 'Got LicenseError "%s" while scanning %s' % (e, path)
257 all_licenses_valid = False
259 # Second, check for non-standard license text.
260 files_data = _ReadFile(os.path.join('android_webview', 'tools',
261 'third_party_files_whitelist.txt'))
262 whitelisted_files = []
263 for line in files_data.splitlines():
264 match = re.match(r'([^#\s]+)', line)
266 whitelisted_files.append(match.group(1))
267 licenses_check = _CheckLicenseHeaders(third_party_dirs, whitelisted_files)
269 return licenses_check if all_licenses_valid else ScanResult.Errors
272 def GenerateNoticeFile():
273 """Generates the contents of an Android NOTICE file for the third-party code.
274 This is used by the snapshot tool.
276 The contents of the NOTICE file.
279 third_party_dirs = _FindThirdPartyDirs()
281 # Don't forget Chromium's LICENSE file
282 content = [_ReadFile('LICENSE')]
284 # We provide attribution for all third-party directories.
285 # TODO(steveblock): Limit this to only code used by the WebView binary.
286 for directory in sorted(third_party_dirs):
287 metadata = licenses.ParseDir(directory, REPOSITORY_ROOT,
288 require_license_file=False)
289 license_file = metadata['License File']
290 if license_file and license_file != licenses.NOT_SHIPPED:
291 content.append(_ReadFile(license_file))
293 return '\n'.join(content)
297 class FormatterWithNewLines(optparse.IndentedHelpFormatter):
298 def format_description(self, description):
299 paras = description.split('\n')
300 formatted_paras = [textwrap.fill(para, self.width) for para in paras]
301 return '\n'.join(formatted_paras) + '\n'
303 parser = optparse.OptionParser(formatter=FormatterWithNewLines(),
304 usage='%prog [options]')
305 parser.description = (__doc__ +
307 ' scan Check licenses.\n' \
308 ' notice Generate Android NOTICE file on stdout. \n' \
309 ' incompatible_directories Scan for incompatibly'
310 ' licensed directories.')
311 (_, args) = parser.parse_args()
314 return ScanResult.Errors
316 if args[0] == 'scan':
317 scan_result = _Scan()
318 if scan_result == ScanResult.Ok:
321 elif args[0] == 'notice':
322 print GenerateNoticeFile()
324 elif args[0] == 'incompatible_directories':
325 incompatible_directories = GetUnknownIncompatibleDirectories()
326 if incompatible_directories:
327 print ("Incompatibly licensed directories found:\n" +
328 "\n".join(sorted(incompatible_directories)))
329 return ScanResult.Errors
333 return ScanResult.Errors
335 if __name__ == '__main__':