3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """Generate an HTML file containing license info for all installed packages.
9 Documentation on this script is also available here:
10 http://www.chromium.org/chromium-os/licensing-for-chromiumos-developers
12 End user (i.e. package owners) documentation is here:
13 http://www.chromium.org/chromium-os/licensing-for-chromiumos-package-owners
16 For this script to work, you must have built the architecture
17 this is being run against, _after_ you've last run repo sync.
18 Otherwise, it will query newer source code and then fail to work on packages
19 that are out of date in your build.
24 sudo rm -rf /build/$BOARD
25 cd ~/trunk/src/scripts
26 # If you wonder why we need to build Chromium OS just to run
27 # `emerge -p -v virtual/target-os` on it, we don't.
28 # However, later we run ebuild unpack, and this will apply patches and run
29 # configure. Configure will fail due to aclocal macros missing in
30 # /build/x86-alex/usr/share/aclocal (those are generated during build).
31 # This will take about 10mn on a Z620.
32 ./build_packages --board=$BOARD --nowithautotest --nowithtest --nowithdev
34 cd ~/trunk/chromite/licensing
35 # This removes left over packages from an earlier build that could cause
37 eclean-$BOARD packages
38 %(prog)s [--debug] [--all-packages] --board $BOARD [-o o.html] 2>&1 | tee out
40 The workflow above is what you would do to generate a licensing file by hand
41 given a chromeos tree.
42 Note that building packages now creates a license.yaml fork in the package
43 which you can see with
44 qtbz2 -x -O /build/x86-alex/packages/dev-util/libc-bench-0.0.1-r8.tbz2 |
45 qxpak -x -O - license.yaml
46 This gets automatically installed in
47 /build/x86-alex/var/db/pkg/dev-util/libc-bench-0.0.1-r8/license.yaml
49 Unless you run with --generate, the script will now gather those license
50 bits and generate a license file from there.
51 License bits for each package are generated by default from
52 src/scripts/hooks/install/gen-package-licenses.sh which gets run automatically
53 by emerge as part of a package build (by running this script with
54 --hook /path/to/tmp/portage/build/tree/for/that/package
56 If license bits are missing, they are generated on the fly if you were running
57 with sudo. If you didn't use sudo, this on the fly late generation will fail
58 and act as a warning that your prebuilts were missing package build time
61 You can check the licenses and/or generate a HTML file for a list of
62 packages using --package or -p:
63 %(prog)s --package "dev-libs/libatomic_ops-7.2d" --package
64 "net-misc/wget-1.14" --board $BOARD -o out.html
66 Note that you'll want to use --generate to force regeneration of the licensing
67 bits from a package source you may have just modified but not rebuilt.
69 If you want to check licensing against all ChromeOS packages, you should
70 run ./build_packages --board=$BOARD to build everything and then run
71 this script with --all-packages.
73 By default, when no package is specified, this script processes all
74 packages for $BOARD. The output HTML file is meant to update
75 http://src.chromium.org/viewvc/chrome/trunk/src/chrome/browser/resources/ +
76 chromeos/about_os_credits.html?view=log
77 (gclient config svn://svn.chromium.org/chrome/trunk/src)
78 For an example CL, see https://codereview.chromium.org/13496002/
80 The detailed process is listed below.
82 * Check out the branch you intend to generate the HTML file for. Use
83 the internal manifest for this purpose.
84 repo init -b <branch_name> -u <URL>
86 The list of branches (e.g. release-R33-5116.B) are available here:
87 https://chromium.googlesource.com/chromiumos/manifest/+refs
89 * Generate the HTML file by following the steps mentioned
90 previously. Check whether your changes are valid with:
91 bin/diff_license_html output.html-M33 output.html-M34
94 * Update the about_os_credits.html in the svn repository. Create a CL
95 and upload it for review.
96 gcl change <change_name>
97 gcl upload <change_name>
99 When uploading, you may get a warning for file being too large to
100 upload. In this case, your CL can still be reviewed. Always include
101 the diff in your commit message so that the reviewers know what the
102 changes are. You can add reviewers on the review page by clicking on
103 "Edit issue". (A quick reference:
104 http://www.chromium.org/developers/quick-reference)
106 Make sure you click on 'Publish+Mail Comments' after adding reviewers
107 (the review URL looks like this https://codereview.chromium.org/183883018/ ).
109 * After receiving LGTMs, commit your change with 'gcl commit <change_name>'.
111 If you don't get this in before the freeze window, it'll need to be merged into
112 the branch being released, which is done by adding a Merge-Requested label.
113 Once it's been updated to "Merge-Approved" by a TPM, please merge into the
114 required release branch. You can ask karen@ for merge approve help.
115 Example: http://crbug.com/221281
117 Note however that this is only during the transition period.
118 build-image will be modified to generate the license for each board and save
119 the file in /opt/google/chrome/resources/about_os_credits.html or as defined
120 in http://crbug.com/271832 .
130 from chromite.buildbot import constants
131 from chromite.buildbot import portage_utilities
132 from chromite.lib import commandline
133 from chromite.lib import cros_build_lib
134 from chromite.lib import osutils
136 # We are imported by src/repohooks/pre-upload.py in a non chroot environment
137 # where yaml may not be there, so we don't error on that since it's not needed
146 # See http://crbug.com/207004 for discussion.
147 PER_PKG_LICENSE_DIR = '/var/db/pkg'
149 STOCK_LICENSE_DIRS = [
150 os.path.join(constants.SOURCE_ROOT,
151 'src/third_party/portage-stable/licenses'),
154 # There are licenses for custom software we got and isn't part of
156 CUSTOM_LICENSE_DIRS = [
157 os.path.join(constants.SOURCE_ROOT,
158 'src/third_party/chromiumos-overlay/licenses'),
161 COPYRIGHT_ATTRIBUTION_DIR = (
163 constants.SOURCE_ROOT,
164 'src/third_party/chromiumos-overlay/licenses/copyright-attribution'))
166 # Virtual packages don't need to have a license and often don't, so we skip them
167 # chromeos-base contains google platform packages that are covered by the
168 # general license at top of tree, so we skip those too.
169 SKIPPED_CATEGORIES = [
174 # Fix these packages by adding a real license in the code.
175 # You should not skip packages just because the license scraping doesn't
176 # work. Stick those special cases into PACKAGE_LICENSES.
177 # Packages should only be here because they are sub/split packages already
178 # covered by the license of the main package.
180 # These are Chrome-OS-specific packages, copyright BSD-Google
181 'sys-kernel/chromeos-kernel', # already manually credit Linux
184 LICENSE_NAMES_REGEX = [
186 r'^copyright[.]txt$',
187 r'^copyright[.]regex$', # llvm
190 r'^licensing.*$', # libatomic_ops
191 r'^ipa_font_license_agreement_v1[.]0[.]txt$', # ja-ipafonts
192 r'^PKG-INFO$', # copyright assignment for
193 # some python packages
194 # (netifaces, unittest2)
197 # These are _temporary_ license mappings for packages that do not have a valid
198 # shared/custom license, or LICENSE file we can use.
199 # Once this script runs earlier (during the package build process), it will
200 # block new source without a LICENSE file if the ebuild contains a license
201 # that requires copyright assignment (BSD and friends).
202 # At that point, new packages will get fixed to include LICENSE instead of
203 # adding workaround mappings like those below.
204 # The way you now fix copyright attribution cases create a custom file with the
205 # right license directly in COPYRIGHT_ATTRIBUTION_DIR.
207 # TODO: replace the naive license parsing code in this script with a hook
208 # into portage's license parsing. See http://crbug.com/348779
210 # Chrome (the browser) is complicated, it has a morphing license that is
211 # either BSD-Google, or BSD-Google,Google-TOS depending on how it was
212 # built. We bypass this problem for now by hardcoding the Google-TOS bit as
213 # per ChromeOS with non free bits
214 'chromeos-base/chromeos-chrome': ['BSD-Google', 'Google-TOS'],
216 # Currently the code cannot parse LGPL-3 || ( LGPL-2.1 MPL-1.1 )
217 'dev-python/pycairo': ['LGPL-3', 'LGPL-2.1'],
220 # Any license listed list here found in the ebuild will make the code look for
221 # license files inside the package source code in order to get copyright
222 # attribution from them.
223 COPYRIGHT_ATTRIBUTION_LICENSES = [
224 'BSD', # requires distribution of copyright notice
225 'BSD-2', # so does BSD-2 http://opensource.org/licenses/BSD-2-Clause
226 'BSD-3', # and BSD-3? http://opensource.org/licenses/BSD-3-Clause
228 'BSD-with-attribution',
230 'MIT-with-advertising',
234 # The following licenses are not invalid or to show as a less helpful stock
235 # license, but it's better to look in the source code for a more specific
236 # license if there is one, but not an error if no better one is found.
237 # Note that you don't want to set just anything here since any license here
238 # will be included once in stock form and a second time in custom form if
239 # found (there is no good way to know that a license we found on disk is the
240 # better version of the stock version, so we show both).
241 LOOK_IN_SOURCE_LICENSES = [
242 'as-is', # The stock license is very vague, source always has more details.
243 'PSF-2', # The custom license in python is more complete than the template.
245 # As far as I know, we have no requirement to do copyright attribution for
246 # these licenses, but the license included in the code has slightly better
247 # information than the stock Gentoo one (including copyright attribution).
248 'BZIP2', # Single use license, do copyright attribution.
249 'OFL', # Almost single use license, do copyright attribution.
250 'OFL-1.1', # Almost single use license, do copyright attribution.
251 'UoI-NCSA', # Only used by NSCA, might as well show their custom copyright.
254 # This used to provide overrides. I can't find a valid reason to add any more
256 PACKAGE_HOMEPAGES = {
258 # 'x11-proto/glproto': ['http://www.x.org/'],
261 # These are tokens found in LICENSE= in an ebuild that aren't licenses we
262 # can actually read from disk.
263 # You should not use this to blacklist real licenses.
265 ')', # Ignore OR tokens from LICENSE="|| ( LGPL-2.1 MPL-1.1 )"
270 TMPL = 'about_credits.tmpl'
271 ENTRY_TMPL = 'about_credits_entry.tmpl'
272 SHARED_LICENSE_TMPL = 'about_credits_shared_license_entry.tmpl'
275 # This is called directly by src/repohooks/pre-upload.py
276 def GetLicenseTypesFromEbuild(ebuild_path):
277 """Returns a list of license types from the ebuild file.
279 This function does not always return the correct list, but it is
280 faster than using portageq for not having to access chroot. It is
281 intended to be used for tasks such as presubmission checks.
284 ebuild_path: ebuild to read.
287 list of licenses read from ebuild.
290 ValueError: ebuild errors.
292 ebuild_env_tmpl = """
293 has() { [[ " ${*:2} " == *" $1 "* ]]; }
295 local overlay_list="%(overlay_list)s"
296 local eclass overlay f
298 has ${eclass} ${_INHERITED_} && continue
299 _INHERITED_+=" ${eclass}"
300 for overlay in %(overlay_list)s; do
301 f="${overlay}/eclass/${eclass}.eclass"
302 if [[ -e ${f} ]]; then
311 # TODO: the overlay_list hard-coded here should be changed to look
312 # at the current overlay, and then the master overlays. E.g. for an
313 # ebuild file in overlay-parrot, we will look at parrot overlay
314 # first, and then look at portage-stable and chromiumos, which are
315 # listed as masters in overlay-parrot/metadata/layout.conf.
317 'ebuild': ebuild_path,
318 'overlay_list': '%s %s' % (
319 os.path.join(constants.SOURCE_ROOT,
320 'src/third_party/chromiumos-overlay'),
321 os.path.join(constants.SOURCE_ROOT,
322 'src/third_party/portage-stable'))
325 with tempfile.NamedTemporaryFile(bufsize=0) as f:
326 osutils.WriteFile(f.name, ebuild_env_tmpl % tmpl_env)
327 env = osutils.SourceEnvironment(
328 f.name, whitelist=['LICENSE'], ifs=' ', multiline=True)
330 if not env.get('LICENSE'):
331 raise ValueError('No LICENSE found in the ebuild.')
332 if re.search(r'[,;]', env['LICENSE']):
334 'LICENSE field in the ebuild should be whitespace-limited.')
336 return env['LICENSE'].split()
339 class PackageLicenseError(Exception):
340 """Thrown if something fails while getting license information for a package.
342 This will cause the processing to error in the end.
346 class PackageInfo(object):
347 """Package info containers, mostly for storing licenses."""
354 # Array of scanned license texts.
355 self.license_text_scanned = []
361 # Looks something like this
362 # /mnt/host/source/src/
363 # third_party/portage-stable/net-misc/rsync/rsync-3.0.8.ebuild
364 self.ebuild_path = None
366 # Array of license names retrieved from ebuild or override in this code.
367 self.ebuild_license_names = []
369 # This contains licenses names we can read from Gentoo or custom licenses.
370 # These are supposed to be shared licenses (i.e. licenses referenced by
371 # more then one package), but after all processing, we may find out that
372 # some are only used once and they get taken out of the shared pool and
373 # pasted directly in the sole package that was using them (see
374 # GenerateHTMLLicenseOutput).
375 self.license_names = set()
377 # We set this if the ebuild has a BSD/MIT like license that requires
378 # scanning for a LICENSE file in the source code, or a static mapping
379 # in PACKAGE_LICENSES. Not finding one once this is set, is fatal.
380 self.need_copyright_attribution = False
381 # This flag just says we'd like to include licenses from the source, but
382 # not finding any is not fatal.
383 self.scan_source_for_licenses = False
385 # After reading basic package information, we can mark the package as
386 # one to skip in licensing.
389 # If we failed to get licensing for this package, mark it as such so that
390 # it can be flagged when the full license file is being generated.
391 self.licensing_failed = False
393 # If we are called from a hook, we grab package info from the soure tree.
394 # This is also used as a flag to know whether we should do package work
395 # based on an installed package, or one that is being built and we got
396 # called from the hook.
397 self.build_source_tree = None
400 def fullnamerev(self):
401 s = '%s-%s' % (self.fullname, self.version)
403 s += '-r%s' % self.revision
408 return '%s/%s' % (self.category, self.name)
411 def license_dump_path(self):
412 """e.g. /build/x86-alex//var/db/pkg/sys-apps/dtc-1.4.0/license.yaml."""
413 return "%s/%s/%s/license.yaml" % (cros_build_lib.GetSysroot(self.board),
414 PER_PKG_LICENSE_DIR, self.fullnamerev)
416 def _BuildInfo(self, filename):
417 filename = '%s/build-info/%s' % (self.build_source_tree, filename)
418 # Buildinfo properties we read are in US-ASCII, not Unicode.
420 bi = open(filename).read().rstrip()
421 # Some properties like HOMEPAGE may be absent.
426 def _RunEbuildPhases(self, phases):
427 """Run a list of ebuild phases on an ebuild.
430 phases: list of phases like ['clean', 'fetch'] or ['unpack'].
433 ebuild command output
436 return cros_build_lib.RunCommand(
437 ['ebuild-%s' % self.board, self.ebuild_path] + phases, print_cmd=debug,
438 redirect_stdout=True)
440 def _GetOverrideLicense(self):
441 """Look in COPYRIGHT_ATTRIBUTION_DIR for license with copyright attribution.
443 For dev-util/bsdiff-4.3-r5, the code will look for
444 dev-util/bsdiff-4.3-r5
448 It is ok to have more than one bsdiff license file, and an empty file acts
449 as a rubout (i.e. an empty dev-util/bsdiff-4.4 will shadow dev-util/bsdiff
450 and tell the licensing code to look in the package source for a license
451 instead of using dev-util/bsdiff as an override).
454 False (no license found) or a multiline license string.
457 # dev-util/bsdiff-4.3-r5 -> bsdiff-4.3-r5
458 filename = os.path.basename(self.fullnamerev)
459 license_path = os.path.join(COPYRIGHT_ATTRIBUTION_DIR,
460 os.path.dirname(self.fullnamerev))
461 pv = portage_utilities.SplitPV(filename)
462 pv_no_rev = '%s-%s' % (pv.package, pv.version_no_rev)
463 for filename in (pv.pv, pv_no_rev, pv.package):
464 file_path = os.path.join(license_path, filename)
465 logging.debug("Looking for override copyright attribution license in %s",
467 if os.path.exists(file_path):
469 # /../merlin/trunk/src/third_party/chromiumos-overlay/../dev-util/bsdiff
471 # chromiumos-overlay/../dev-util/bsdiff
472 short_dir_path = os.path.join(*file_path.rsplit(os.path.sep, 5)[1:])
473 license_read = "Copyright Attribution License %s:\n\n" % short_dir_path
474 license_read += ReadUnknownEncodedFile(
475 file_path, "read copyright attribution license")
480 def _ExtractLicenses(self):
481 """Scrounge for text licenses in the source of package we'll unpack.
483 This is only called if we couldn't get usable licenses from the ebuild,
484 or one of them is BSD/MIT like which forces us to look for a file with
485 copyright attribution in the source code itself.
487 First, we have a shortcut where we scan COPYRIGHT_ATTRIBUTION_DIR to see if
488 we find a license for this package. If so, we use that.
489 Typically it'll be used if the unpacked source does not have the license
490 that we're required to display for copyright attribution (in some cases it's
491 plain absent, in other cases, it could be in a filename we don't look for).
493 Otherwise, we scan the unpacked source code for what looks like license
494 files as defined in LICENSE_NAMES_REGEX.
497 AssertionError: on runtime errors
498 PackageLicenseError: couldn't find copyright attribution file.
500 license_override = self._GetOverrideLicense()
502 self.license_text_scanned = [license_override]
505 if self.build_source_tree:
506 workdir = "%s/work" % self.build_source_tree
508 self._RunEbuildPhases(['clean', 'fetch'])
509 output = self._RunEbuildPhases(['unpack']).output.splitlines()
510 # Output is spammy, it looks like this:
511 # * gc-7.2d.tar.gz RMD160 SHA1 SHA256 size ;-) ... [ ok ]
512 # * checking gc-7.2d.tar.gz ;-) ... [ ok ]
513 # * Running stacked hooks for pre_pkg_setup
514 # * sysroot_build_bin_dir ...
516 # * Running stacked hooks for pre_src_unpack
517 # * python_multilib_setup ...
519 # >>> Unpacking source...
520 # >>> Unpacking gc-7.2d.tar.gz to /build/x86-alex/tmp/po/[...]ps-7.2d/work
521 # >>> Source unpacked in /build/x86-alex/tmp/portage/[...]ops-7.2d/work
522 # So we only keep the last 2 lines, the others we don't care about.
523 output = [line for line in output if line[0:3] == ">>>" and
524 line != ">>> Unpacking source..."]
528 args = ['portageq-%s' % self.board, 'envvar', 'PORTAGE_TMPDIR']
529 result = cros_build_lib.RunCommand(args, print_cmd=debug,
530 redirect_stdout=True)
531 tmpdir = result.output.splitlines()[0]
532 # tmpdir gets something like /build/daisy/tmp/
533 workdir = os.path.join(tmpdir, 'portage', self.fullnamerev, 'work')
535 if not os.path.exists(workdir):
536 raise AssertionError("Unpack of %s didn't create %s. Version mismatch" %
537 (self.fullnamerev, workdir))
539 # You may wonder how deep should we go?
540 # In case of packages with sub-packages, it could be deep.
541 # Let's just be safe and get everything we can find.
542 # In the case of libatomic_ops, it's actually required to look deep
543 # to find the MIT license:
544 # dev-libs/libatomic_ops-7.2d/work/gc-7.2/libatomic_ops/doc/LICENSING.txt
545 args = ['find', workdir, '-type', 'f']
546 result = cros_build_lib.RunCommand(args, print_cmd=debug,
547 redirect_stdout=True).output.splitlines()
548 # Truncate results to look like this: swig-2.0.4/COPYRIGHT
549 files = [x[len(workdir):].lstrip('/') for x in result]
552 # When we scan a source tree managed by git, this can contain license
553 # files that are not part of the source. Exclude those.
554 # (e.g. .git/refs/heads/licensing)
557 basename = os.path.basename(name)
558 # Looking for license.* brings up things like license.gpl, and we
559 # never want a GPL license when looking for copyright attribution,
560 # so we skip them here. We also skip regexes that can return
561 # license.py (seen in some code).
562 if re.search(r".*GPL.*", basename) or re.search(r"\.py$", basename):
564 for regex in LICENSE_NAMES_REGEX:
565 if re.search(regex, basename, re.IGNORECASE):
566 license_files.append(name)
569 if not license_files:
570 if self.need_copyright_attribution:
572 %s: unable to find usable license.
573 Typically this will happen because the ebuild says it's MIT or BSD, but there
574 was no license file that this script could find to include along with a
575 copyright attribution (required for BSD/MIT).
577 If this is Google source, please change
582 If not, go investigate the unpacked source in %s,
583 and find which license to assign. Once you found it, you should copy that
584 license to a file under %s
585 (or you can modify LICENSE_NAMES_REGEX to pickup a license file that isn't
586 being scraped currently).""",
587 self.fullnamerev, workdir, COPYRIGHT_ATTRIBUTION_DIR)
588 raise PackageLicenseError()
590 # We can get called for a license like as-is where it's preferable
591 # to find a better one in the source, but not fatal if we didn't.
592 logging.info("Was not able to find a better license for %s "
593 "in %s to replace the more generic one from ebuild",
594 self.fullnamerev, workdir)
596 # Examples of multiple license matches:
597 # dev-lang/swig-2.0.4-r1: swig-2.0.4/COPYRIGHT swig-2.0.4/LICENSE
598 # dev-libs/glib-2.32.4-r1: glib-2.32.4/COPYING pkg-config-0.26/COPYING
599 # dev-libs/libnl-3.2.14: libnl-doc-3.2.14/COPYING libnl-3.2.14/COPYING
600 # dev-libs/libpcre-8.30-r2: pcre-8.30/LICENCE pcre-8.30/COPYING
601 # dev-libs/libusb-0.1.12-r6: libusb-0.1.12/COPYING libusb-0.1.12/LICENSE
602 # dev-libs/pyzy-0.1.0-r1: db/COPYING pyzy-0.1.0/COPYING
603 # net-misc/strongswan-5.0.2-r4: strongswan-5.0.2/COPYING
604 # strongswan-5.0.2/LICENSE
605 # sys-process/procps-3.2.8_p11: debian/copyright procps-3.2.8/COPYING
606 logging.info('License(s) for %s: %s', self.fullnamerev,
607 ' '.join(license_files))
608 for license_file in sorted(license_files):
609 # Joy and pink ponies. Some license_files are encoded as latin1 while
610 # others are utf-8 and of course you can't know but only guess.
611 license_path = os.path.join(workdir, license_file)
612 license_txt = ReadUnknownEncodedFile(license_path, "Adding License")
614 self.license_text_scanned += [
615 "Scanned Source License %s:\n\n%s" % (license_file, license_txt)]
617 # We used to clean up here, but there have been many instances where
618 # looking at unpacked source to see where the licenses were, was useful
619 # so let's disable this for now
620 # self._RunEbuildPhases(['clean'])
622 def GetPackageInfo(self, fullnamewithrev):
623 """Populate PackageInfo with package license, and homepage.
625 self.ebuild_license_names will not be filled if the package is skipped
626 or if there was an issue getting data from the ebuild.
627 self.license_names will only get the licenses that we can paste
629 scan_source_for_licenses will be set if we should unpack the source to look
631 if need_copyright_attribution is also set, not finding a license in the
632 source is fatal (PackageLicenseError will get raised).
635 fullnamewithrev: e.g. dev-libs/libatomic_ops-7.2d
638 AssertionError: on runtime errors
640 if not fullnamewithrev:
641 if not self.build_source_tree:
642 raise AssertionError("Cannot continue without full name or source tree")
643 fullnamewithrev = "%s/%s" % (self._BuildInfo("CATEGORY"),
644 self._BuildInfo("PF"))
645 logging.debug("Computed package name %s from %s", fullnamewithrev,
646 self.build_source_tree)
649 cpv = portage_utilities.SplitCPV(fullnamewithrev)
650 # A bad package can either raise a TypeError exception or return None,
651 # so we catch both cases.
655 raise AssertionError("portage couldn't find %s, missing version number?" %
658 (self.category, self.name, self.version, self.revision) = (
659 cpv.category, cpv.package, cpv.version_no_rev, cpv.rev)
661 if self.revision is not None:
662 self.revision = str(self.revision).lstrip('r')
663 if self.revision == '0':
666 if self.category in SKIPPED_CATEGORIES:
667 logging.info("%s in SKIPPED_CATEGORIES, skip package", self.fullname)
671 if self.fullname in SKIPPED_PACKAGES:
672 logging.info("%s in SKIPPED_PACKAGES, skip package", self.fullname)
676 def _ReadEbuildInfo(self):
677 """Populate package info from an ebuild retrieved via equery."""
678 # By default, equery returns the latest version of the package. A
679 # build may have used an older version than what is currently
680 # available in the source tree (a build dependency can be pinned
681 # to an older version of a package for compatibility
682 # reasons). Therefore we need to tell equery that we want the
683 # exact version number used in the image build as opposed to the
684 # latest available in the source tree.
685 args = ['equery-%s' % self.board, 'which', self.fullnamerev]
687 path = cros_build_lib.RunCommand(args, print_cmd=debug,
688 redirect_stdout=True).output.strip()
692 raise AssertionError('GetEbuildPath for %s failed.\n'
693 'Is your tree clean? Delete %s and rebuild' %
695 cros_build_lib.GetSysroot(board=self.board)))
696 logging.debug("%s -> %s", " ".join(args), path)
698 if not os.access(path, os.F_OK):
699 raise AssertionError("Can't access %s", path)
701 self.ebuild_path = path
703 args = ['portageq-%s' % self.board, 'metadata',
704 cros_build_lib.GetSysroot(board=self.board), 'ebuild',
705 self.fullnamerev, 'HOMEPAGE', 'LICENSE']
706 tmp = cros_build_lib.RunCommand(args, print_cmd=debug,
707 redirect_stdout=True)
708 lines = tmp.output.splitlines()
710 # portageq metadata /build/x86-alex ebuild net-misc/wget-1.12-r2 \
713 # http://www.gnu.org/software/wget/
715 (self.homepages, self.ebuild_license_names) = (
716 lines[0].split(), lines[1].split())
718 def GetLicenses(self):
719 """Get licenses from the ebuild field and the unpacked source code.
721 Some packages have static license mappings applied to them that get
722 retrieved from the ebuild.
724 For others, we figure out whether the package source should be scanned to
725 add licenses found there.
728 AssertionError: on runtime errors
729 PackageLicenseError: couldn't find license in ebuild and source.
731 if self.build_source_tree:
732 self.homepages = self._BuildInfo("HOMEPAGE").split()
733 self.ebuild_license_names = self._BuildInfo("LICENSE").split()
735 self._ReadEbuildInfo()
737 if self.fullname in PACKAGE_HOMEPAGES:
738 self.homepages = PACKAGE_HOMEPAGES[self.fullname]
740 # Packages with missing licenses or licenses that need mapping (like
741 # BSD/MIT) are hardcoded here:
742 if self.fullname in PACKAGE_LICENSES:
743 self.ebuild_license_names = PACKAGE_LICENSES[self.fullname]
744 logging.info("Static license mapping for %s: %s", self.fullnamerev,
745 ",".join(self.ebuild_license_names))
747 logging.info("Read licenses for %s: %s", self.fullnamerev,
748 ",".join(self.ebuild_license_names))
750 # Lots of packages in chromeos-base have their license set to BSD instead
752 new_license_names = []
753 for license_name in self.ebuild_license_names:
754 # TODO: temp workaround for http;//crbug.com/348750 , remove when the bug
756 if (license_name == "BSD" and
757 self.fullnamerev.startswith("chromeos-base/")):
758 license_name = "BSD-Google"
760 "Fixed BSD->BSD-Google for %s because it's in chromeos-base. "
761 "Please fix the LICENSE field in the ebuild", self.fullnamerev)
762 # TODO: temp workaround for http;//crbug.com/348749 , remove when the bug
764 if license_name == "Proprietary":
765 license_name = "Google-TOS"
767 "Fixed Proprietary -> Google-TOS for %s. "
768 "Please fix the LICENSE field in the ebuild", self.fullnamerev)
769 new_license_names.append(license_name)
770 self.ebuild_license_names = new_license_names
772 # The ebuild license field can look like:
773 # LICENSE="GPL-3 LGPL-3 Apache-2.0" (this means AND, as in all 3)
774 # for third_party/portage-stable/app-admin/rsyslog/rsyslog-5.8.11.ebuild
775 # LICENSE="|| ( LGPL-2.1 MPL-1.1 )"
776 # for third_party/portage-stable/x11-libs/cairo/cairo-1.8.8.ebuild
778 # The parser isn't very smart and only has basic support for the
779 # || ( X Y ) OR logic to do the following:
780 # In order to save time needlessly unpacking packages and looking or a
781 # cleartext license (which is really a crapshoot), if we have a license
782 # like BSD that requires looking for copyright attribution, but we can
783 # chose another license like GPL, we do that.
785 if not self.ebuild_license_names:
786 logging.error("%s: no license found in ebuild. FIXME!", self.fullnamerev)
787 # In a bind, you could comment this out. I'm making the output fail to
788 # get your attention since this error really should be fixed, but if you
789 # comment out the next line, the script will try to find a license inside
791 raise PackageLicenseError()
793 # This is not invalid, but the parser can't deal with it, so if it ever
794 # happens, error out to tell the programmer to do something.
795 # dev-python/pycairo-1.10.0-r4: LGPL-3 || ( LGPL-2.1 MPL-1.1 )
796 if "||" in self.ebuild_license_names[1:]:
797 logging.error("%s: Can't parse || in the middle of a license: %s",
798 self.fullnamerev, ' '.join(self.ebuild_license_names))
799 raise PackageLicenseError()
801 or_licenses_and_one_is_no_attribution = False
802 # We do a quick early pass first so that the longer pass below can
804 for license_name in [x for x in self.ebuild_license_names
805 if x not in LICENCES_IGNORE]:
806 # Here we have an OR case, and one license that we can use stock, so
807 # we remember that in order to be able to skip license attributions if
808 # any were in the OR.
809 if (self.ebuild_license_names[0] == "||" and
810 license_name not in COPYRIGHT_ATTRIBUTION_LICENSES):
811 or_licenses_and_one_is_no_attribution = True
813 for license_name in [x for x in self.ebuild_license_names
814 if x not in LICENCES_IGNORE]:
815 # Licenses like BSD or MIT can't be used as is because they do not contain
816 # copyright self. They have to be replaced by copyright file given in the
817 # source code, or manually mapped by us in PACKAGE_LICENSES
818 if license_name in COPYRIGHT_ATTRIBUTION_LICENSES:
819 # To limit needless efforts, if a package is BSD or GPL, we ignore BSD
820 # and use GPL to avoid scanning the package, but we can only do this if
821 # or_licenses_and_one_is_no_attribution has been set above.
822 # This ensures that if we have License: || (BSD3 BSD4), we will
823 # look in the source.
824 if or_licenses_and_one_is_no_attribution:
825 logging.info("%s: ignore license %s because ebuild LICENSES had %s",
826 self.fullnamerev, license_name,
827 ' '.join(self.ebuild_license_names))
829 logging.info("%s: can't use %s, will scan source code for copyright",
830 self.fullnamerev, license_name)
831 self.need_copyright_attribution = True
832 self.scan_source_for_licenses = True
834 self.license_names.add(license_name)
835 # We can't display just 2+ because it only contains text that says to
837 if license_name == 'GPL-2+':
838 self.license_names.add('GPL-2')
839 if license_name == 'LGPL-2+':
840 self.license_names.add('LGPL-2')
842 if license_name in LOOK_IN_SOURCE_LICENSES:
843 logging.info("%s: Got %s, will try to find better license in source...",
844 self.fullnamerev, license_name)
845 self.scan_source_for_licenses = True
847 if self.license_names:
848 logging.info('%s: using stock|cust license(s) %s',
849 self.fullnamerev, ','.join(self.license_names))
851 # If the license(s) could not be found, or one requires copyright
852 # attribution, dig in the source code for license files:
854 # Read licenses from ebuild for net-dialup/ppp-2.4.5-r3: BSD,GPL-2
855 # We need get the substitution file for BSD and add it to GPL.
856 if self.scan_source_for_licenses:
857 self._ExtractLicenses()
859 # This shouldn't run, but leaving as sanity check.
860 if not self.license_names and not self.license_text_scanned:
861 raise AssertionError("Didn't find usable licenses for %s" %
865 class Licensing(object):
866 """Do the actual work of extracting licensing info and outputting html."""
868 def __init__(self, board, package_fullnames, gen_licenses):
871 # List of stock and custom licenses referenced in ebuilds. Used to
872 # print a report. Dict value says which packages use that license.
875 # Licenses are supposed to be generated at package build time and be
876 # ready for us, but in case they're not, they can be generated.
877 self.gen_licenses = gen_licenses
879 # This keeps track of whether we have an incomplete license file due to
880 # package errors during parsing.
881 # Any non empty list at the end shows the list of packages that caused
883 self.incomplete_packages = []
885 self.package_text = {}
886 self.entry_template = None
888 # We need to have a dict for the list of packages objects, index by package
889 # fullnamerev, so that when we scan our licenses at the end, and find out
890 # some shared licenses are only used by one package, we can access that
891 # package object by name, and add the license directly in that object.
893 self._package_fullnames = package_fullnames
896 def sorted_licenses(self):
897 return sorted(self.licenses.keys(), key=str.lower)
899 def _SaveLicenseDump(self, pkg):
900 if pkg.build_source_tree:
901 save_file = "%s/build-info/license.yaml" % pkg.build_source_tree
903 save_file = pkg.license_dump_path
904 logging.debug("Saving license to %s", save_file)
905 save_dir = os.path.dirname(save_file)
906 if not os.path.isdir(save_dir):
907 os.makedirs(save_dir, 0755)
908 with open(save_file, "w") as f:
910 for key, value in pkg.__dict__.items():
911 yaml_dump.append([key, value])
912 f.write(yaml.dump(yaml_dump))
914 def _LoadLicenseDump(self, pkg):
915 save_file = pkg.license_dump_path
916 logging.debug("Getting license from %s for %s", save_file, pkg.name)
917 with open(save_file, "r") as f:
918 # yaml.safe_load barfs on unicode it output, but we don't really need it.
919 yaml_dump = yaml.load(f)
920 for key, value in yaml_dump:
921 pkg.__dict__[key] = value
923 def LicensedPackages(self, license_name):
924 """Return list of packages using a given license."""
925 return self.licenses[license_name]
927 def LoadPackageInfo(self, board):
928 """Populate basic package info for all packages from their ebuild."""
929 for package_name in self._package_fullnames:
932 pkg.GetPackageInfo(package_name)
933 self.packages[package_name] = pkg
935 def HookPackageProcess(self, pkg_build_path):
936 """Different entry point to populate a packageinfo.
938 This is called instead of LoadPackageInfo when called by a package build.
941 pkg_build_path: unpacked being built by emerge.
944 pkg.build_source_tree = pkg_build_path
945 pkg.GetPackageInfo(None)
948 self._SaveLicenseDump(pkg)
950 def ProcessPackageLicenses(self):
951 """Iterate through all packages provided and gather their licenses.
953 GetLicenses will scrape licenses from the code and/or gather stock license
954 names. We gather the list of stock and custom ones for later processing.
956 Do not call this after adding virtual packages with AddExtraPkg.
958 for package_name in self.packages:
959 pkg = self.packages[package_name]
961 if self.gen_licenses:
962 logging.info("Package %s is in skip list", package_name)
964 # If we do a licensing run expecting to get licensing objects from
965 # an image build, virtual packages will be missing such objects
966 # because virtual packages do not get the install hook run at build
967 # time. Because this script may not have permissions to write in the
968 # /var/db/ directory, we don't want it to generate useless license
969 # bits for virtual packages. As a result, ignore virtual packages
971 if pkg.category == "virtual":
972 logging.debug("Ignoring %s virtual package", package_name)
975 # Other skipped packages get dumped with incomplete info and the skip flag
976 if not os.path.exists(pkg.license_dump_path) and not self.gen_licenses:
977 logging.warning(">>> License for %s is missing, creating now <<<",
979 if not os.path.exists(pkg.license_dump_path) or self.gen_licenses:
983 except PackageLicenseError:
984 pkg.licensing_failed = True
985 # We dump packages where licensing failed too.
986 self._SaveLicenseDump(pkg)
988 # To debug the code, we force the data to be re-read from the dumps
989 # instead of reusing what we may have in memory.
990 for package_name in self.packages:
991 pkg = self.packages[package_name]
992 self._LoadLicenseDump(pkg)
993 logging.debug("loaded dump for %s", pkg.fullnamerev)
995 logging.info("Package %s is in skip list", pkg.fullnamerev)
996 if pkg.licensing_failed:
997 logging.info("Package %s failed licensing", pkg.fullnamerev)
998 self.incomplete_packages += [pkg.fullnamerev]
1000 def AddExtraPkg(self, pkg_data):
1001 """Allow adding pre-created virtual packages.
1003 GetLicenses will not work on them, so add them after having run
1007 pkg_data: array of package data as defined below
1010 pkg.board = self.board
1011 pkg.category = pkg_data[0]
1012 pkg.name = pkg_data[1]
1013 pkg.version = pkg_data[2]
1014 pkg.homepages = pkg_data[3] # this is a list
1015 pkg.license_names = pkg_data[4] # this is also a list
1016 pkg.ebuild_license_names = pkg_data[4]
1017 self.packages[pkg.fullnamerev] = pkg
1019 # Called directly by src/repohooks/pre-upload.py
1021 def FindLicenseType(license_name):
1022 """Says if a license is stock Gentoo, custom, or doesn't exist."""
1024 for directory in STOCK_LICENSE_DIRS:
1025 path = '%s/%s' % (directory, license_name)
1026 if os.path.exists(path):
1027 return "Gentoo Package Stock"
1029 for directory in CUSTOM_LICENSE_DIRS:
1030 path = '%s/%s' % (directory, license_name)
1031 if os.path.exists(path):
1034 raise AssertionError("""
1035 license %s could not be found in %s
1036 If the license in the ebuild is correct,
1037 a) a stock license should be added to portage-stable/licenses :
1038 running `cros_portage_upgrade` inside of the chroot should clone this repo
1040 https://chromium.googlesource.com/chromiumos/overlays/portage/+/gentoo
1041 find the new licenses under licenses, and add them to portage-stable/licenses
1043 b) if it's a non gentoo package with a custom license, you can copy that license
1044 to third_party/chromiumos-overlay/licenses/
1046 Try re-running the script with -p cat/package-ver --generate
1047 after fixing the license.""" %
1049 '\n'.join(STOCK_LICENSE_DIRS + CUSTOM_LICENSE_DIRS))
1053 def ReadSharedLicense(license_name):
1054 """Read and return stock or cust license file specified in an ebuild."""
1057 for directory in STOCK_LICENSE_DIRS + CUSTOM_LICENSE_DIRS:
1058 path = os.path.join(directory, license_name)
1059 if os.path.exists(path):
1064 return ReadUnknownEncodedFile(license_path, "read license")
1066 raise AssertionError("license %s could not be found in %s"
1068 '\n'.join(STOCK_LICENSE_DIRS +
1069 CUSTOM_LICENSE_DIRS))
1073 def EvaluateTemplate(template, env):
1074 """Expand a template with vars like {{foo}} using a dict of expansions."""
1075 # TODO switch to stock python templates.
1076 for key, val in env.iteritems():
1077 template = template.replace('{{%s}}' % key, val)
1080 def _GeneratePackageLicenseText(self, pkg):
1081 """Concatenate all licenses related to a pkg.
1083 This means a combination of ebuild shared licenses and licenses read from
1084 the pkg source tree, if any.
1087 pkg: PackageInfo object
1090 AssertionError: on runtime errors
1093 for license_text_scanned in pkg.license_text_scanned:
1094 license_text.append(license_text_scanned)
1095 license_text.append('%s\n' % ('-=' * 40))
1097 license_pointers = []
1098 # sln: shared license name.
1099 for sln in pkg.license_names:
1100 # Says whether it's a stock gentoo or custom license.
1101 license_type = self.FindLicenseType(sln)
1102 license_pointers.append(
1103 "<li><a href='#%s'>%s License %s</a></li>" % (
1104 sln, license_type, sln))
1106 # This should get caught earlier, but one extra check.
1107 if not license_text + license_pointers:
1108 raise AssertionError('Ended up with no license_text for %s', pkg.name)
1111 'name': "%s-%s" % (pkg.name, pkg.version),
1112 'url': cgi.escape(pkg.homepages[0]) if pkg.homepages else '',
1113 'licenses_txt': cgi.escape('\n'.join(license_text)) or '',
1114 'licenses_ptr': '\n'.join(license_pointers) or '',
1116 self.package_text[pkg] = self.EvaluateTemplate(self.entry_template, env)
1118 def GenerateHTMLLicenseOutput(self, output_file,
1119 output_template=TMPL,
1120 entry_template=ENTRY_TMPL,
1121 license_template=SHARED_LICENSE_TMPL):
1122 """Generate the combined html license file used in ChromeOS.
1125 output_file: resulting HTML license output.
1126 output_template: template for the entire HTML file.
1127 entry_template: template for per package entries.
1128 license_template: template for shared license entries.
1130 self.entry_template = ReadUnknownEncodedFile(entry_template)
1131 sorted_license_txt = []
1133 # Keep track of which licenses are used by which packages.
1134 for pkg in self.packages.values():
1135 if pkg.skip or pkg.licensing_failed:
1137 for sln in pkg.license_names:
1138 self.licenses.setdefault(sln, []).append(pkg.fullnamerev)
1140 # Find licenses only used once, and roll them in the package that uses them.
1141 # We use keys() because licenses is modified in the loop, so we can't use
1143 for sln in self.licenses.keys():
1144 if len(self.licenses[sln]) == 1:
1145 pkg_fullnamerev = self.licenses[sln][0]
1146 logging.info("Collapsing shared license %s into single use license "
1147 "(only used by %s)", sln, pkg_fullnamerev)
1148 license_type = self.FindLicenseType(sln)
1149 license_txt = self.ReadSharedLicense(sln)
1150 single_license = "%s License %s:\n\n%s" % (license_type, sln,
1152 pkg = self.packages[pkg_fullnamerev]
1153 pkg.license_text_scanned.append(single_license)
1154 pkg.license_names.remove(sln)
1155 del self.licenses[sln]
1157 for pkg in sorted(self.packages.values(),
1158 key=lambda x: (x.name.lower(), x.version, x.revision)):
1160 logging.debug("Skipping package %s", pkg.fullnamerev)
1162 if pkg.licensing_failed:
1163 logging.debug("Package %s failed licensing, skipping", pkg.fullnamerev)
1165 self._GeneratePackageLicenseText(pkg)
1166 sorted_license_txt += [self.package_text[pkg]]
1168 # Now generate the bottom of the page that will contain all the shared
1169 # licenses and a list of who is pointing to them.
1170 license_template = ReadUnknownEncodedFile(license_template)
1173 for license_name in self.sorted_licenses:
1175 'license_name': license_name,
1176 'license': cgi.escape(self.ReadSharedLicense(license_name)),
1177 'license_type': self.FindLicenseType(license_name),
1178 'license_packages': ' '.join(self.LicensedPackages(license_name)),
1180 licenses_txt += [self.EvaluateTemplate(license_template, env)]
1182 file_template = ReadUnknownEncodedFile(output_template)
1184 'entries': '\n'.join(sorted_license_txt),
1185 'licenses': '\n'.join(licenses_txt),
1187 osutils.WriteFile(output_file,
1188 self.EvaluateTemplate(file_template, env).encode('UTF-8'))
1191 def ListInstalledPackages(board, all_packages=False):
1192 """Return a list of all packages installed for a particular board."""
1194 # If all_packages is set to True, all packages visible in the build
1195 # chroot are used to generate the licensing file. This is not what you want
1196 # for a release license file, but it's a way to run licensing checks against
1198 # If it's set to False, it will only generate a licensing file that contains
1199 # packages used for a release build (as determined by the dependencies for
1200 # virtual/target-os).
1203 # The following returns all packages that were part of the build tree
1204 # (many get built or used during the build, but do not get shipped).
1205 # Note that it also contains packages that are in the build as
1206 # defined by build_packages but not part of the image we ship.
1207 args = ["equery-%s" % board, "list", "*"]
1208 packages = cros_build_lib.RunCommand(args, print_cmd=debug,
1209 redirect_stdout=True
1210 ).output.splitlines()
1212 # The following returns all packages that were part of the build tree
1213 # (many get built or used during the build, but do not get shipped).
1214 # Note that it also contains packages that are in the build as
1215 # defined by build_packages but not part of the image we ship.
1216 args = ["emerge-%s" % board, "--with-bdeps=y", "--usepkgonly",
1217 "--emptytree", "--pretend", "--color=n", "virtual/target-os"]
1218 emerge = cros_build_lib.RunCommand(args, print_cmd=debug,
1219 redirect_stdout=True).output.splitlines()
1220 # Another option which we've decided not to use, is bdeps=n. This outputs
1221 # just the packages we ship, but does not packages that were used to build
1222 # them, including a package like flex which generates a .a that is included
1223 # and shipped in ChromeOS.
1224 # We've decided to credit build packages, even if we're not legally required
1225 # to (it's always nice to do), and that way we get corner case packages like
1226 # flex. This is why we use bdep=y and not bdep=n.
1229 # [binary R ] x11-libs/libva-1.1.1 to /build/x86-alex/
1230 pkg_rgx = re.compile(r'\[[^]]+R[^]]+\] (.+) to /build/.*')
1231 # If we match something else without the 'R' like
1232 # [binary U ] chromeos-base/pepper-flash-13.0.0.133-r1 [12.0.0.77-r1]
1233 # this is bad and we should die on this.
1234 pkg_rgx2 = re.compile(r'(\[[^]]+\] .+) to /build/.*')
1236 match = pkg_rgx.search(line)
1237 match2 = pkg_rgx2.search(line)
1239 packages.append(match.group(1))
1241 raise AssertionError("Package incorrectly installed, try eclean-%s" %
1242 board, "\n%s" % match2.group(1))
1247 def _HandleIllegalXMLChars(text):
1248 """Handles illegal XML Characters.
1250 XML 1.0 acceptable character range:
1251 Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | \
1254 This function finds all illegal characters in the text and filters
1255 out all whitelisted characters (e.g. ^L).
1258 text: text to examine.
1261 Filtered |text| and a list of non-whitelisted illegal characters found.
1263 whitelist_re = re.compile(u'[\x0c]')
1264 text = whitelist_re.sub('', text)
1265 # illegal_chars_re includes all illegal characters (whitelisted or
1266 # not), so we can expand the whitelist without modifying this line.
1267 illegal_chars_re = re.compile(
1268 u'[\x00-\x08\x0b\x0c\x0e-\x1F\uD800-\uDFFF\uFFFE\uFFFF]')
1269 return (text, illegal_chars_re.findall(text))
1272 def ReadUnknownEncodedFile(file_path, logging_text=None):
1273 """Read a file of unknown encoding (UTF-8 or latin) by trying in sequence.
1276 file_path: what to read.
1277 logging_text: what to display for logging depending on file read.
1280 File content, possibly converted from latin1 to UTF-8.
1283 Assertion error: if non-whitelisted illegal XML characters
1284 are found in the file.
1285 ValueError: returned if we get invalid XML.
1288 with codecs.open(file_path, encoding="utf-8") as c:
1291 logging.info("%s %s (UTF-8)", logging_text, file_path)
1292 except UnicodeDecodeError:
1293 with codecs.open(file_path, encoding="latin1") as c:
1296 logging.info("%s %s (latin1)", logging_text, file_path)
1298 file_txt, char_list = _HandleIllegalXMLChars(file_txt)
1301 raise ValueError('Illegal XML characters %s found in %s.' %
1302 (char_list, file_path))
1307 def NonHookMain(opts):
1308 """Do the work when we're not called as a hook."""
1310 board, all_packages, gen_licenses, output_file = (
1311 opts.board, opts.all_packages, opts.gen_licenses, opts.output)
1312 packages_mode = bool(opts.package)
1315 raise AssertionError("No board given (--board)")
1316 logging.info("Using board %s.", board)
1318 builddir = os.path.join(cros_build_lib.GetSysroot(board=board),
1320 if not os.path.exists(builddir):
1321 raise AssertionError(
1322 "FATAL: %s missing.\n"
1323 "Did you give the right board and build that tree?" % builddir)
1324 if not output_file and not gen_licenses:
1325 logging.warning("You are not generating licenses and you didn't ask for "
1326 "output. As a result this script will do nothing useful.")
1327 license_dir = "%s/%s/" % (cros_build_lib.GetSysroot(board),
1328 PER_PKG_LICENSE_DIR)
1329 if not os.path.exists(license_dir):
1330 raise AssertionError("FATAL: %s missing.\n" % license_dir)
1332 if gen_licenses and os.geteuid() != 0:
1333 raise AssertionError("Run with sudo if you use --generate-licenses.")
1336 packages = opts.package
1338 packages = ListInstalledPackages(board, all_packages)
1340 raise AssertionError('FATAL: Could not get any packages for board %s' %
1342 logging.debug("Initial Package list to work through:\n%s",
1343 '\n'.join(sorted(packages)))
1344 licensing = Licensing(board, packages, gen_licenses)
1345 licensing.LoadPackageInfo(board)
1346 logging.debug("Package list to skip:\n%s",
1347 '\n'.join([p for p in sorted(packages)
1348 if licensing.packages[p].skip]))
1349 logging.debug("Package list left to work through:\n%s",
1350 '\n'.join([p for p in sorted(packages)
1351 if not licensing.packages[p].skip]))
1352 licensing.ProcessPackageLicenses()
1353 if not packages_mode:
1354 # We add 2 virtual packages as well as 2 boot packages that are included
1355 # with some hardware, but not in the image or package list.
1357 ['x11-base', 'X.Org', '1.9.3', ['http://www.x.org/'], ['X']],
1358 ['sys-kernel', 'Linux', '2.6', ['http://www.kernel.org/'], ['GPL-2']],
1359 ['sys-boot', 'u-boot', '2013.06', ['http://www.denx.de/wiki/U-Boot'],
1361 ['sys-boot', 'coreboot', '2013.04', ['http://www.coreboot.org/'],
1364 licensing.AddExtraPkg(extra_pkg)
1367 licensing.GenerateHTMLLicenseOutput(output_file)
1369 if licensing.incomplete_packages:
1370 raise AssertionError("""
1371 DO NOT USE OUTPUT!!!
1372 Some packages are missing due to errors, please look at errors generated
1374 List of packages with errors:
1376 """ % '\n'.join(licensing.incomplete_packages))
1380 # pylint: disable=W0603
1382 # pylint: enable=W0603
1384 parser = commandline.ArgumentParser(usage=__doc__)
1385 parser.add_argument("-b", "--board",
1386 help="which board to run for, like x86-alex")
1387 parser.add_argument("-p", "--package", action="append", default=[],
1388 help="check the license of the package, e.g.,"
1389 "dev-libs/libatomic_ops-7.2d")
1390 parser.add_argument("-a", "--all-packages", action="store_true",
1391 dest="all_packages",
1392 help="Run licensing against all packages in the "
1394 parser.add_argument("-g", "--generate-licenses", action="store_true",
1395 dest="gen_licenses",
1396 help="Generate licensing bits for each package before "
1397 "making license file\n(default is to use build time "
1399 parser.add_argument("-k", "--hook", type="path", dest="hook",
1400 help="Hook mode takes a single package and outputs its "
1401 "license on stdout. Give $PORTAGE_BUILDDIR as argument.")
1402 parser.add_argument("-o", "--output", type="path",
1403 help="which html file to create with output")
1404 opts = parser.parse_args(args)
1407 hook_path = opts.hook
1410 # This get called from src/scripts/hooks/install/gen-package-licenses.sh
1412 licensing = Licensing(None, None, True)
1413 licensing.HookPackageProcess(hook_path)