1 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Perform various tasks related to updating Portage packages."""
7 from __future__ import print_function
12 import parallel_emerge
13 import portage # pylint: disable=F0401
18 from chromite.cbuildbot import constants
19 from chromite.lib import commandline
20 from chromite.lib import cros_build_lib
21 from chromite.lib import osutils
22 from chromite.lib import operation
23 from chromite.lib import upgrade_table as utable
24 from chromite.scripts import merge_package_status as mps
26 # pylint: disable=E1101,W0201
28 oper = operation.Operation('cros_portage_upgrade')
30 NOT_APPLICABLE = 'N/A'
31 WORLD_TARGET = 'world'
34 # Arches we care about -- we actively develop/support/ship.
35 STANDARD_BOARD_ARCHS = set(('amd64', 'arm', 'x86'))
37 # Files we do not include in our upgrades by convention.
38 BLACKLISTED_FILES = set(['Manifest', 'ChangeLog*'])
41 """Class to accumulate package info during upgrade process.
43 This class is basically a formalized dictionary.
47 'category', # Package category only
48 # TODO(mtennant): Rename 'cpv' to 'curr_cpv' or similar.
49 'cpv', # Current full cpv (revision included)
50 'cpv_cmp_upstream', # 0 = current, >0 = outdated, <0 = futuristic
51 'latest_upstream_cpv', # Latest (non-stable ok) upstream cpv
52 'overlay', # Overlay package currently in
53 'package', # category/package_name
54 'package_name', # The 'p' in 'cpv'
55 'package_ver', # The 'pv' in 'cpv'
56 'slot', # Current package slot
57 'stable_upstream_cpv', # Latest stable upstream cpv
58 'state', # One of utable.UpgradeTable.STATE_*
59 'upgraded_cpv', # If upgraded, it is to this cpv
60 'upgraded_unmasked', # Boolean. If upgraded_cpv, indicates if unmasked.
61 'upstream_cpv', # latest/stable upstream cpv according to request
62 'user_arg', # Original user arg for this pkg, if applicable
63 'version_rev', # Just revision (e.g. 'r1'). '' if no revision
66 # Any deriving classes must maintain this cumulative attribute list.
67 __attrlist__ = __slots__
69 def __init__(self, **kwargs):
70 """Initialize all attributes to None unless specified in |kwargs|."""
71 for attr in self.__attrlist__:
72 setattr(self, attr, kwargs.get(attr))
74 def __eq__(self, other):
75 """Equality support. Used in unittests."""
77 if type(self) != type(other):
81 for attr in self.__attrlist__:
82 if getattr(self, attr, no_attr) != getattr(other, attr, no_attr):
87 def __ne__(self, other):
88 """Inequality support for completeness."""
89 return not self == other
91 class Upgrader(object):
92 """A class to perform various tasks related to updating Portage packages."""
94 PORTAGE_GIT_URL = '%s/chromiumos/overlays/portage.git' % (
95 constants.EXTERNAL_GOB_URL)
96 ORIGIN_GENTOO = 'origin/gentoo'
98 UPSTREAM_OVERLAY_NAME = 'portage'
99 UPSTREAM_TMP_REPO = os.environ.get(constants.SHARED_CACHE_ENVVAR)
100 if UPSTREAM_TMP_REPO is not None:
101 UPSTREAM_TMP_REPO = '%s/cros_portage_upgrade' % UPSTREAM_TMP_REPO
103 UPSTREAM_TMP_REPO = '/tmp'
104 UPSTREAM_TMP_REPO += '/' + UPSTREAM_OVERLAY_NAME
106 STABLE_OVERLAY_NAME = 'portage-stable'
107 CROS_OVERLAY_NAME = 'chromiumos-overlay'
108 CATEGORIES_FILE = 'profiles/categories'
109 HOST_BOARD = 'amd64-host'
110 OPT_SLOTS = ('amend', 'csv_file', 'force', 'no_upstream_cache', 'rdeps',
111 'upgrade', 'upgrade_deep', 'upstream', 'unstable_ok', 'verbose',
114 EQUERY_CMD = 'equery'
115 EMERGE_CMD = 'emerge'
116 PORTAGEQ_CMD = 'portageq'
117 BOARD_CMDS = set([EQUERY_CMD, EMERGE_CMD, PORTAGEQ_CMD])
119 __slots__ = ['_amend', # Boolean to use --amend with upgrade commit
120 '_args', # Commandline arguments (all portage targets)
121 '_curr_arch', # Architecture for current board run
122 '_curr_board', # Board for current board run
123 '_curr_table', # Package status for current board run
124 '_cros_overlay', # Path to chromiumos-overlay repo
125 '_csv_file', # File path for writing csv output
126 '_deps_graph', # Dependency graph from portage
127 '_force', # Force upgrade even when version already exists
128 '_local_only', # Skip network traffic
129 '_missing_eclass_re',# Regexp for missing eclass in equery
130 '_outdated_eclass_re',# Regexp for outdated eclass in equery
131 '_emptydir', # Path to temporary empty directory
132 '_master_archs', # Set. Archs of tables merged into master_table
133 '_master_cnt', # Number of tables merged into master_table
134 '_master_table', # Merged table from all board runs
135 '_no_upstream_cache', # Boolean. Delete upstream cache when done
136 '_porttree', # Reference to portage porttree object
137 '_rdeps', # Boolean, if True pass --root-deps=rdeps
138 '_stable_repo', # Path to portage-stable
139 '_stable_repo_categories', # Categories from profiles/categories
140 '_stable_repo_stashed', # True if portage-stable has a git stash
141 '_stable_repo_status', # git status report at start of run
142 '_targets', # Processed list of portage targets
143 '_upgrade', # Boolean indicating upgrade requested
144 '_upgrade_cnt', # Num pkg upgrades in this run (all boards)
145 '_upgrade_deep', # Boolean indicating upgrade_deep requested
146 '_upstream', # Path to upstream portage repo
147 '_unstable_ok', # Boolean to allow unstable upstream also
148 '_verbose', # Boolean
151 def __init__(self, options):
152 self._args = options.packages
153 self._targets = mps.ProcessTargets(self._args)
155 self._master_table = None
157 self._master_archs = set()
158 self._upgrade_cnt = 0
160 self._stable_repo = os.path.join(options.srcroot, 'third_party',
161 self.STABLE_OVERLAY_NAME)
162 # This can exist in two spots; the tree, or the cache.
164 self._cros_overlay = os.path.join(options.srcroot, 'third_party',
165 self.CROS_OVERLAY_NAME)
167 # Save options needed later.
168 for opt in self.OPT_SLOTS:
169 setattr(self, '_' + opt, getattr(options, opt, None))
171 self._porttree = None
172 self._emptydir = None
173 self._deps_graph = None
175 # Pre-compiled regexps for speed.
176 self._missing_eclass_re = re.compile(r'(\S+\.eclass) could not be '
178 self._outdated_eclass_re = re.compile(r'Call stack:\n'
179 r'(?:.*?\s+\S+,\sline.*?\n)*'
180 r'.*?\s+(\S+\.eclass),\s+line')
182 def _IsInUpgradeMode(self):
183 """Return True if running in upgrade mode."""
184 return self._upgrade or self._upgrade_deep
186 def _SaveStatusOnStableRepo(self):
187 """Get the 'git status' for everything in |self._stable_repo|.
189 The results are saved in a dict at self._stable_repo_status where each key
190 is a file path rooted at |self._stable_repo|, and the value is the status
191 for that file as returned by 'git status -s'. (e.g. 'A' for 'Added').
193 result = self._RunGit(self._stable_repo, ['status', '-s'],
194 redirect_stdout=True)
195 if result.returncode == 0:
197 for line in result.output.strip().split('\n'):
201 linesplit = line.split()
202 (status, path) = linesplit[0], linesplit[1]
204 # Handle a rename as separate 'D' and 'A' statuses. Example line:
205 # R path/to/foo-1.ebuild -> path/to/foo-2.ebuild
207 statuses[linesplit[3]] = 'A'
209 statuses[path] = status
211 self._stable_repo_status = statuses
213 raise RuntimeError('Unable to run "git status -s" in %s:\n%s' %
214 (self._stable_repo, result.output))
216 self._stable_repo_stashed = False
218 def _LoadStableRepoCategories(self):
219 """Load |self._stable_repo|/profiles/categories into set."""
221 self._stable_repo_categories = set()
222 cat_file_path = os.path.join(self._stable_repo, self.CATEGORIES_FILE)
223 with open(cat_file_path, 'r') as f:
227 self._stable_repo_categories.add(line)
229 def _WriteStableRepoCategories(self):
230 """Write |self._stable_repo_categories| to profiles/categories."""
232 categories = sorted(self._stable_repo_categories)
233 cat_file_path = os.path.join(self._stable_repo, self.CATEGORIES_FILE)
234 with open(cat_file_path, 'w') as f:
235 f.writelines('\n'.join(categories))
237 self._RunGit(self._stable_repo, ['add', self.CATEGORIES_FILE])
239 def _CheckStableRepoOnBranch(self):
240 """Raise exception if |self._stable_repo| is not on a branch now."""
241 result = self._RunGit(self._stable_repo, ['branch'], redirect_stdout=True)
242 if result.returncode == 0:
243 for line in result.output.split('\n'):
244 match = re.search(r'^\*\s+(.+)$', line)
246 # Found current branch, see if it is a real branch.
247 branch = match.group(1)
248 if branch != '(no branch)':
250 raise RuntimeError('To perform upgrade, %s must be on a branch.' %
253 raise RuntimeError('Unable to determine whether %s is on a branch.' %
256 def _PkgUpgradeRequested(self, pinfo):
257 """Return True if upgrade of pkg in |pinfo| was requested by user."""
258 if self._upgrade_deep:
262 return bool(pinfo.user_arg)
267 def _FindBoardArch(board):
268 """Return the architecture for a given board name."""
269 # Host is a special case
270 if board == Upgrader.HOST_BOARD:
273 # Leverage Portage 'portageq' tool to do this.
274 cmd = ['portageq-%s' % board, 'envvar', 'ARCH']
275 cmd_result = cros_build_lib.RunCommand(
276 cmd, print_cmd=False, redirect_stdout=True)
277 if cmd_result.returncode == 0:
278 return cmd_result.output.strip()
283 def _GetPreOrderDepGraphPackage(deps_graph, package, pkglist, visited):
284 """Collect packages from |deps_graph| into |pkglist| in pre-order."""
285 if package in visited:
288 for parent in deps_graph[package]['provides']:
289 Upgrader._GetPreOrderDepGraphPackage(deps_graph, parent, pkglist, visited)
290 pkglist.append(package)
293 def _GetPreOrderDepGraph(deps_graph):
294 """Return packages from |deps_graph| in pre-order."""
297 for package in deps_graph:
298 Upgrader._GetPreOrderDepGraphPackage(deps_graph, package, pkglist,
303 def _CmpCpv(cpv1, cpv2):
304 """Returns standard cmp result between |cpv1| and |cpv2|.
306 If one cpv is None then the other is greater.
308 if cpv1 is None and cpv2 is None:
314 return portage.versions.pkgcmp(portage.versions.pkgsplit(cpv1),
315 portage.versions.pkgsplit(cpv2))
318 def _GetCatPkgFromCpv(cpv):
319 """Returns category/package_name from a full |cpv|.
321 If |cpv| is incomplete, may return only the package_name.
323 If package_name cannot be determined, return None.
328 # Result is None or (cat, pn, version, rev)
329 result = portage.versions.catpkgsplit(cpv)
331 # This appears to be a quirk of portage? Category string == 'null'.
332 if result[0] is None or result[0] == 'null':
334 return '%s/%s' % (result[0], result[1])
339 def _GetVerRevFromCpv(cpv):
340 """Returns just the version-revision string from a full |cpv|."""
344 # Result is None or (cat, pn, version, rev)
345 result = portage.versions.catpkgsplit(cpv)
347 (version, rev) = result[2:4]
349 return '%s-%s' % (version, rev)
356 def _GetEbuildPathFromCpv(cpv):
357 """Returns the relative path to ebuild for |cpv|."""
361 # Result is None or (cat, pn, version, rev)
362 result = portage.versions.catpkgsplit(cpv)
364 (cat, pn, _version, _rev) = result
365 ebuild = cpv.replace(cat + '/', '') + '.ebuild'
366 return os.path.join(cat, pn, ebuild)
370 def _RunGit(self, cwd, command, redirect_stdout=False,
371 combine_stdout_stderr=False):
372 """Runs git |command| (a list of command tokens) in |cwd|.
374 This leverages the cros_build_lib.RunCommand function. The
375 |redirect_stdout| and |combine_stdout_stderr| arguments are
376 passed to that function.
378 Returns a Result object as documented by cros_build_lib.RunCommand.
379 Most usefully, the result object has a .output attribute containing
380 the output from the command (if |redirect_stdout| was True).
382 # This disables the vi-like output viewer for commands like 'git show'.
383 extra_env = {'GIT_PAGER': 'cat'}
384 cmdline = ['git'] + command
385 return cros_build_lib.RunCommand(
386 cmdline, cwd=cwd, extra_env=extra_env, print_cmd=self._verbose,
387 redirect_stdout=redirect_stdout,
388 combine_stdout_stderr=combine_stdout_stderr)
390 def _SplitEBuildPath(self, ebuild_path):
391 """Split a full ebuild path into (overlay, cat, pn, pv)."""
392 (ebuild_path, _ebuild) = os.path.splitext(ebuild_path)
393 (ebuild_path, pv) = os.path.split(ebuild_path)
394 (ebuild_path, pn) = os.path.split(ebuild_path)
395 (ebuild_path, cat) = os.path.split(ebuild_path)
396 (ebuild_path, overlay) = os.path.split(ebuild_path)
397 return (overlay, cat, pn, pv)
399 def _GenPortageEnvvars(self, arch, unstable_ok, portdir=None,
400 portage_configroot=None):
401 """Returns dictionary of envvars for running portage tools.
403 If |arch| is set, then ACCEPT_KEYWORDS will be included and set
404 according to |unstable_ok|.
406 PORTDIR is set to |portdir| value, if not None.
407 PORTAGE_CONFIGROOT is set to |portage_configroot| value, if not None.
412 envvars['ACCEPT_KEYWORDS'] = arch + ' ~' + arch
414 envvars['ACCEPT_KEYWORDS'] = arch
416 if portdir is not None:
417 envvars['PORTDIR'] = portdir
418 # Since we are clearing PORTDIR, we also have to clear PORTDIR_OVERLAY
419 # as most of those repos refer to the "normal" PORTDIR and will dump a
420 # lot of warnings if it can't be found.
421 envvars['PORTDIR_OVERLAY'] = portdir
422 if portage_configroot is not None:
423 envvars['PORTAGE_CONFIGROOT'] = portage_configroot
427 def _FindUpstreamCPV(self, pkg, unstable_ok=False):
428 """Returns latest cpv in |_upstream| that matches |pkg|.
430 The |pkg| argument can specify as much or as little of the full CPV
431 syntax as desired, exactly as accepted by the Portage 'equery' command.
432 To find whether an exact version exists upstream specify the full
433 CPV. To find the latest version specify just the category and package
436 Results are filtered by architecture keyword using |self._curr_arch|.
437 By default, only ebuilds stable on that arch will be accepted. To
438 accept unstable ebuilds, set |unstable_ok| to True.
440 Returns upstream cpv, if found.
442 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok,
443 portdir=self._upstream,
444 portage_configroot=self._emptydir)
446 # Point equery to the upstream source to get latest version for keywords.
447 equery = ['equery', 'which', pkg]
448 cmd_result = cros_build_lib.RunCommand(
449 equery, extra_env=envvars, print_cmd=self._verbose,
450 error_code_ok=True, redirect_stdout=True, combine_stdout_stderr=True)
452 if cmd_result.returncode == 0:
453 ebuild_path = cmd_result.output.strip()
454 (_overlay, cat, _pn, pv) = self._SplitEBuildPath(ebuild_path)
455 return os.path.join(cat, pv)
459 def _GetBoardCmd(self, cmd):
460 """Return the board-specific version of |cmd|, if applicable."""
461 if cmd in self.BOARD_CMDS:
462 # Host "board" is a special case.
463 if self._curr_board != self.HOST_BOARD:
464 return '%s-%s' % (cmd, self._curr_board)
468 def _AreEmergeable(self, cpvlist):
469 """Indicate whether cpvs in |cpvlist| can be emerged on current board.
471 This essentially runs emerge with the --pretend option to verify
472 that all dependencies for these package versions are satisfied.
475 Tuple with two elements:
476 [0] True if |cpvlist| can be emerged.
477 [1] Output from the emerge command.
479 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok=False)
480 emerge = self._GetBoardCmd(self.EMERGE_CMD)
481 cmd = [emerge, '-p'] + ['=' + cpv for cpv in cpvlist]
482 result = cros_build_lib.RunCommand(
483 cmd, error_code_ok=True, extra_env=envvars, print_cmd=False,
484 redirect_stdout=True, combine_stdout_stderr=True)
486 return (result.returncode == 0, ' '.join(cmd), result.output)
488 def _FindCurrentCPV(self, pkg):
489 """Returns current cpv on |_curr_board| that matches |pkg|, or None."""
490 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok=False)
492 equery = self._GetBoardCmd(self.EQUERY_CMD)
493 cmd = [equery, '-C', 'which', pkg]
494 cmd_result = cros_build_lib.RunCommand(
495 cmd, error_code_ok=True, extra_env=envvars, print_cmd=False,
496 redirect_stdout=True, combine_stdout_stderr=True)
498 if cmd_result.returncode == 0:
499 ebuild_path = cmd_result.output.strip()
500 (_overlay, cat, _pn, pv) = self._SplitEBuildPath(ebuild_path)
501 return os.path.join(cat, pv)
505 def _SetUpgradedMaskBits(self, pinfo):
506 """Set pinfo.upgraded_unmasked."""
507 cpv = pinfo.upgraded_cpv
508 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok=False)
510 equery = self._GetBoardCmd('equery')
511 cmd = [equery, '-qCN', 'list', '-F', '$mask|$cpv:$slot', '-op', cpv]
512 result = cros_build_lib.RunCommand(
513 cmd, error_code_ok=True, extra_env=envvars, print_cmd=False,
514 redirect_stdout=True, combine_stdout_stderr=True)
516 output = result.output
517 if result.returncode:
518 raise RuntimeError('equery failed on us:\n %s\noutput:\n %s'
519 % (' '.join(cmd), output))
521 # Expect output like one of these cases (~ == unstable, M == masked):
522 # ~|sys-fs/fuse-2.7.3:0
523 # |sys-fs/fuse-2.7.3:0
524 # M |sys-fs/fuse-2.7.3:0
525 # M~|sys-fs/fuse-2.7.3:0
526 for line in output.split('\n'):
527 mask = line.split('|')[0]
529 pinfo.upgraded_unmasked = 'M' != mask[0]
532 raise RuntimeError('Unable to determine whether %s is stable from equery:\n'
533 ' %s\noutput:\n %s' % (cpv, ' '.join(cmd), output))
535 def _VerifyEbuildOverlay(self, cpv, expected_overlay, was_overwrite):
536 """Raises exception if ebuild for |cpv| is not from |expected_overlay|.
538 Essentially, this verifies that the upgraded ebuild in portage-stable
539 is indeed the one being picked up, rather than some other ebuild with
540 the same version in another overlay. Unless |was_overwrite| (see below).
542 If |was_overwrite| then this upgrade was an overwrite of an existing
543 package version (via --force) and it is possible the previous package
544 is still in another overlay (e.g. chromiumos-overlay). In this case,
545 the user should get rid of the other version first.
547 # Further explanation: this check should always pass, but might not
548 # if the copy/upgrade from upstream did not work. This is just a
550 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok=False)
552 equery = self._GetBoardCmd(self.EQUERY_CMD)
553 cmd = [equery, '-C', 'which', '--include-masked', cpv]
554 result = cros_build_lib.RunCommand(
555 cmd, error_code_ok=True, extra_env=envvars, print_cmd=False,
556 redirect_stdout=True, combine_stdout_stderr=True)
558 ebuild_path = result.output.strip()
559 (overlay, _cat, _pn, _pv) = self._SplitEBuildPath(ebuild_path)
560 if overlay != expected_overlay:
562 raise RuntimeError('Upgraded ebuild for %s is not visible because'
563 ' existing ebuild in %s overlay takes precedence\n'
564 'Please remove that ebuild before continuing.' %
567 raise RuntimeError('Upgraded ebuild for %s is not coming from %s:\n'
569 'Please show this error to the build team.' %
570 (cpv, expected_overlay, ebuild_path))
572 def _IdentifyNeededEclass(self, cpv):
573 """Return eclass that must be upgraded for this |cpv|."""
574 # Try to detect two cases:
575 # 1) The upgraded package uses an eclass not in local source, yet.
576 # 2) The upgraded package needs one or more eclasses to also be upgraded.
578 # Use the output of 'equery which'.
579 # If a needed eclass cannot be found, then the output will have lines like:
580 # * ERROR: app-admin/eselect-1.2.15 failed (depend phase):
581 # * bash-completion-r1.eclass could not be found by inherit()
583 # If a needed eclass must be upgraded, the output might have the eclass
584 # in the call stack (... used for long paths):
586 # * ebuild.sh, line 2047: Called source '.../vim-7.3.189.ebuild'
587 # * vim-7.3.189.ebuild, line 7: Called inherit 'vim'
588 # * ebuild.sh, line 1410: Called qa_source '.../vim.eclass'
589 # * ebuild.sh, line 43: Called source '.../vim.eclass'
590 # * vim.eclass, line 40: Called die
591 # * The specific snippet of code:
592 # * die "Unknown EAPI ${EAPI}"
594 envvars = self._GenPortageEnvvars(self._curr_arch, unstable_ok=True)
596 equery = self._GetBoardCmd(self.EQUERY_CMD)
597 cmd = [equery, '-C', '--no-pipe', 'which', cpv]
598 result = cros_build_lib.RunCommand(
599 cmd, error_code_ok=True, extra_env=envvars, print_cmd=False,
600 redirect_stdout=True, combine_stdout_stderr=True)
602 if result.returncode != 0:
603 output = result.output.strip()
605 # _missing_eclass_re works line by line.
606 for line in output.split('\n'):
607 match = self._missing_eclass_re.search(line)
609 eclass = match.group(1)
610 oper.Notice('Determined that %s requires %s' % (cpv, eclass))
613 # _outdated_eclass_re works on the entire output at once.
614 match = self._outdated_eclass_re.search(output)
616 eclass = match.group(1)
617 oper.Notice('Making educated guess that %s requires update of %s' %
623 def _GiveMaskedError(self, upgraded_cpv, emerge_output):
624 """Print error saying that |upgraded_cpv| is masked off.
626 See if hint found in |emerge_output| to improve error emssage.
629 # Expecting emerge_output to have lines like this:
630 # The following mask changes are necessary to proceed:
631 # #required by ... =somecategory/somepackage (some reason)
632 # # /home/mtennant/trunk/src/third_party/chromiumos-overlay/profiles\
633 # /targets/chromeos/package.mask:
637 upgraded_cp = Upgrader._GetCatPkgFromCpv(upgraded_cpv)
638 regexp = re.compile(r'#\s*required by.+=\S+.*\n'
639 r'#\s*(\S+/package\.mask):\s*\n'
640 '[<>=]+%s' % upgraded_cp)
642 match = regexp.search(emerge_output)
644 package_mask = match.group(1)
647 oper.Error('\nUpgraded package "%s" appears to be masked by a line in\n'
649 'Full emerge output is above. Address mask issue, '
650 'then run this again.' %
651 (upgraded_cpv, package_mask))
653 oper.Error('\nUpgraded package "%s" is masked somehow (See full '
654 'emerge output above). Address that and then run this '
655 'again.' % upgraded_cpv)
657 def _PkgUpgradeStaged(self, upstream_cpv):
658 """Return True if package upgrade is already staged."""
659 ebuild_path = Upgrader._GetEbuildPathFromCpv(upstream_cpv)
660 status = self._stable_repo_status.get(ebuild_path, None)
661 if status and 'A' == status:
666 def _AnyChangesStaged(self):
667 """Return True if any local changes are staged in stable repo."""
668 # Don't count files with '??' status - they aren't staged.
669 files = [f for (f, s) in self._stable_repo_status.items() if s != '??']
670 return bool(len(files))
672 def _StashChanges(self):
673 """Run 'git stash save' on stable repo."""
674 # Only one level of stashing expected/supported.
675 self._RunGit(self._stable_repo, ['stash', 'save'],
676 redirect_stdout=True, combine_stdout_stderr=True)
677 self._stable_repo_stashed = True
679 def _UnstashAnyChanges(self):
680 """Unstash any changes in stable repo."""
681 # Only one level of stashing expected/supported.
682 if self._stable_repo_stashed:
683 self._RunGit(self._stable_repo, ['stash', 'pop', '--index'],
684 redirect_stdout=True, combine_stdout_stderr=True)
685 self._stable_repo_stashed = False
687 def _DropAnyStashedChanges(self):
688 """Drop any stashed changes in stable repo."""
689 # Only one level of stashing expected/supported.
690 if self._stable_repo_stashed:
691 self._RunGit(self._stable_repo, ['stash', 'drop'],
692 redirect_stdout=True, combine_stdout_stderr=True)
693 self._stable_repo_stashed = False
695 def _CopyUpstreamPackage(self, upstream_cpv):
696 """Upgrades package in |upstream_cpv| to the version in |upstream_cpv|.
699 The upstream_cpv if the package was upgraded, None otherwise.
701 oper.Notice('Copying %s from upstream.' % upstream_cpv)
703 (cat, pkgname, _version, _rev) = portage.versions.catpkgsplit(upstream_cpv)
704 ebuild = upstream_cpv.replace(cat + '/', '') + '.ebuild'
705 catpkgsubdir = os.path.join(cat, pkgname)
706 pkgdir = os.path.join(self._stable_repo, catpkgsubdir)
707 upstream_pkgdir = os.path.join(self._upstream, cat, pkgname)
709 # Fail early if upstream_cpv ebuild is not found
710 upstream_ebuild_path = os.path.join(upstream_pkgdir, ebuild)
711 if not os.path.exists(upstream_ebuild_path):
712 # Note: this should only be possible during unit tests.
713 raise RuntimeError('Cannot find upstream ebuild at "%s"' %
714 upstream_ebuild_path)
716 # If pkgdir already exists, remove everything in it except Manifest.
717 # Note that git will remove a parent directory when it removes
718 # the last item in the directory.
719 if os.path.exists(pkgdir):
720 items = os.listdir(pkgdir)
721 items = [os.path.join(catpkgsubdir, i) for i in items if i != 'Manifest']
723 args = ['rm', '-rf', '--ignore-unmatch'] + items
724 self._RunGit(self._stable_repo, args, redirect_stdout=True)
725 # Now delete any files that git doesn't know about.
727 osutils.SafeUnlink(os.path.join(self._stable_repo, item))
729 osutils.SafeMakedirs(pkgdir)
731 # Grab all non-blacklisted, non-ebuilds from upstream plus the specific
733 items = os.listdir(upstream_pkgdir)
735 blacklisted = [b for b in BLACKLISTED_FILES
736 if fnmatch.fnmatch(os.path.basename(item), b)]
738 if not item.endswith('.ebuild') or item == ebuild:
739 src = os.path.join(upstream_pkgdir, item)
740 dst = os.path.join(pkgdir, item)
741 if os.path.isdir(src):
742 shutil.copytree(src, dst, symlinks=True)
744 shutil.copy2(src, dst)
746 # Create a new Manifest file for this package.
747 self._CreateManifest(upstream_pkgdir, pkgdir, ebuild)
749 # Now copy any eclasses that this package requires.
750 eclass = self._IdentifyNeededEclass(upstream_cpv)
751 while eclass and self._CopyUpstreamEclass(eclass):
752 eclass = self._IdentifyNeededEclass(upstream_cpv)
756 def _StabilizeEbuild(self, ebuild_path):
757 """Edit keywords to stablize ebuild at |ebuild_path| on current arch."""
758 oper.Notice('Editing %r to mark it stable for everyone' % ebuild_path)
760 # Regexp to search for KEYWORDS="...".
761 keywords_regexp = re.compile(r'^(\s*KEYWORDS=")[^"]*(")', re.MULTILINE)
763 # Read in entire ebuild.
764 content = osutils.ReadFile(ebuild_path)
766 # Replace all KEYWORDS with "*".
767 content = re.sub(keywords_regexp, r'\1*\2', content)
769 # Write ebuild file back out.
770 osutils.WriteFile(ebuild_path, content)
772 def _CreateManifest(self, upstream_pkgdir, pkgdir, ebuild):
773 """Create a trusted Manifest from available Manifests.
775 Combine the current Manifest in |pkgdir| (if it exists) with
776 the Manifest from |upstream_pkgdir| to create a new trusted
777 Manifest. Supplement with 'ebuild manifest' command.
779 It is assumed that a Manifest exists in |upstream_pkgdir|, but
780 there may not be one in |pkgdir|. The new |ebuild| in pkgdir
781 should be used for 'ebuild manifest' command.
783 The algorithm is this:
784 1) Remove all lines in upstream Manifest that duplicate
785 lines in current Manifest.
786 2) Concatenate the result of 1) onto the current Manifest.
787 3) Run 'ebuild manifest' to add to results.
789 upstream_manifest = os.path.join(upstream_pkgdir, 'Manifest')
790 current_manifest = os.path.join(pkgdir, 'Manifest')
792 if os.path.exists(current_manifest):
793 # Determine which files have DIST entries in current_manifest.
795 with open(current_manifest, 'r') as f:
797 tokens = line.split()
798 if len(tokens) > 1 and tokens[0] == 'DIST':
801 # Find DIST lines in upstream manifest not overlapping with current.
803 with open(upstream_manifest, 'r') as f:
805 tokens = line.split()
806 if len(tokens) > 1 and tokens[0] == 'DIST' and tokens[1] not in dists:
807 new_lines.append(line)
809 # Write all new_lines to current_manifest.
811 with open(current_manifest, 'a') as f:
812 f.writelines(new_lines)
814 # Use upstream_manifest as a starting point.
815 shutil.copyfile(upstream_manifest, current_manifest)
817 manifest_cmd = ['ebuild', os.path.join(pkgdir, ebuild), 'manifest']
818 manifest_result = cros_build_lib.RunCommand(
819 manifest_cmd, error_code_ok=True, print_cmd=False,
820 redirect_stdout=True, combine_stdout_stderr=True)
822 if manifest_result.returncode != 0:
823 raise RuntimeError('Failed "ebuild manifest" for upgraded package.\n'
824 'Output of %r:\n%s' %
825 (' '.join(manifest_cmd), manifest_result.output))
828 def _CopyUpstreamEclass(self, eclass):
829 """Upgrades eclass in |eclass| to upstream copy.
831 Does not do the copy if the eclass already exists locally and
832 is identical to the upstream version.
835 True if the copy was done.
837 eclass_subpath = os.path.join('eclass', eclass)
838 upstream_path = os.path.join(self._upstream, eclass_subpath)
839 local_path = os.path.join(self._stable_repo, eclass_subpath)
841 if os.path.exists(upstream_path):
842 if os.path.exists(local_path) and filecmp.cmp(upstream_path, local_path):
845 oper.Notice('Copying %s from upstream.' % eclass)
846 osutils.SafeMakedirs(os.path.dirname(local_path))
847 shutil.copy2(upstream_path, local_path)
848 self._RunGit(self._stable_repo, ['add', eclass_subpath])
851 raise RuntimeError('Cannot find upstream "%s". Looked at "%s"' %
852 (eclass, upstream_path))
854 def _GetPackageUpgradeState(self, pinfo):
855 """Return state value for package in |pinfo|."""
856 # See whether this specific cpv exists upstream.
858 cpv_exists_upstream = bool(cpv and
859 self._FindUpstreamCPV(cpv, unstable_ok=True))
861 # The value in pinfo.cpv_cmp_upstream represents a comparison of cpv
862 # version and the upstream version, where:
863 # 0 = current, >0 = outdated, <0 = futuristic!
865 # Convention is that anything not in portage overlay has been altered.
866 overlay = pinfo.overlay
867 locally_patched = (overlay != NOT_APPLICABLE and
868 overlay != self.UPSTREAM_OVERLAY_NAME and
869 overlay != self.STABLE_OVERLAY_NAME)
870 locally_duplicated = locally_patched and cpv_exists_upstream
872 # Gather status details for this package
873 if pinfo.cpv_cmp_upstream is None:
874 # No upstream cpv to compare to (although this might include a
875 # a restriction to only stable upstream versions). This is concerning
876 # if the package is coming from 'portage' or 'portage-stable' overlays.
877 if locally_patched and pinfo.latest_upstream_cpv is None:
878 state = utable.UpgradeTable.STATE_LOCAL_ONLY
880 state = utable.UpgradeTable.STATE_UPSTREAM_ONLY
882 state = utable.UpgradeTable.STATE_UNKNOWN
883 elif pinfo.cpv_cmp_upstream > 0:
884 if locally_duplicated:
885 state = utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED
886 elif locally_patched:
887 state = utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED
889 state = utable.UpgradeTable.STATE_NEEDS_UPGRADE
890 elif locally_duplicated:
891 state = utable.UpgradeTable.STATE_DUPLICATED
892 elif locally_patched:
893 state = utable.UpgradeTable.STATE_PATCHED
895 state = utable.UpgradeTable.STATE_CURRENT
899 # TODO(mtennant): Generate output from finished table instead.
900 def _PrintPackageLine(self, pinfo):
901 """Print a brief one-line report of package status."""
902 upstream_cpv = pinfo.upstream_cpv
903 if pinfo.upgraded_cpv:
904 action_stat = ' (UPGRADED)'
908 up_stat = {utable.UpgradeTable.STATE_UNKNOWN: ' no package found upstream!',
909 utable.UpgradeTable.STATE_LOCAL_ONLY: ' (exists locally only)',
910 utable.UpgradeTable.STATE_NEEDS_UPGRADE: ' -> %s' % upstream_cpv,
911 utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_PATCHED:
912 ' <-> %s' % upstream_cpv,
913 utable.UpgradeTable.STATE_NEEDS_UPGRADE_AND_DUPLICATED:
914 ' (locally duplicated) <-> %s' % upstream_cpv,
915 utable.UpgradeTable.STATE_PATCHED: ' <- %s' % upstream_cpv,
916 utable.UpgradeTable.STATE_DUPLICATED: ' (locally duplicated)',
917 utable.UpgradeTable.STATE_CURRENT: ' (current)',
920 oper.Info('[%s] %s%s%s' % (pinfo.overlay, pinfo.cpv,
921 up_stat, action_stat))
923 def _AppendPackageRow(self, pinfo):
924 """Add a row to status table for the package in |pinfo|."""
926 upgraded_cpv = pinfo.upgraded_cpv
930 upgraded_ver = Upgrader._GetVerRevFromCpv(upgraded_cpv)
932 # Assemble 'depends on' and 'required by' strings.
933 depsstr = NOT_APPLICABLE
934 usedstr = NOT_APPLICABLE
935 if cpv and self._deps_graph:
936 deps_entry = self._deps_graph[cpv]
937 depslist = sorted(deps_entry['needs'].keys()) # dependencies
938 depsstr = ' '.join(depslist)
939 usedset = deps_entry['provides'] # used by
940 usedlist = sorted([p for p in usedset])
941 usedstr = ' '.join(usedlist)
943 stable_up_ver = Upgrader._GetVerRevFromCpv(pinfo.stable_upstream_cpv)
944 if not stable_up_ver:
945 stable_up_ver = NOT_APPLICABLE
946 latest_up_ver = Upgrader._GetVerRevFromCpv(pinfo.latest_upstream_cpv)
947 if not latest_up_ver:
948 latest_up_ver = NOT_APPLICABLE
950 row = {self._curr_table.COL_PACKAGE: pinfo.package,
951 self._curr_table.COL_SLOT: pinfo.slot,
952 self._curr_table.COL_OVERLAY: pinfo.overlay,
953 self._curr_table.COL_CURRENT_VER: pinfo.version_rev,
954 self._curr_table.COL_STABLE_UPSTREAM_VER: stable_up_ver,
955 self._curr_table.COL_LATEST_UPSTREAM_VER: latest_up_ver,
956 self._curr_table.COL_STATE: pinfo.state,
957 self._curr_table.COL_DEPENDS_ON: depsstr,
958 self._curr_table.COL_USED_BY: usedstr,
959 self._curr_table.COL_TARGET: ' '.join(self._targets),
962 # Only include if upgrade was involved. Table may not have this column
963 # if upgrade was not requested.
965 row[self._curr_table.COL_UPGRADED] = upgraded_ver
967 self._curr_table.AppendRow(row)
969 def _UpgradePackage(self, pinfo):
970 """Gathers upgrade status for pkg, performs upgrade if requested.
972 The upgrade is performed only if the package is outdated and --upgrade
975 The |pinfo| must have the following entries:
976 package, category, package_name
978 Regardless, the following attributes in |pinfo| are filled in:
981 upstream_cpv (one of the above, depending on --stable-only option)
982 upgrade_cpv (if upgrade performed)
985 catpkg = pinfo.package
986 pinfo.stable_upstream_cpv = self._FindUpstreamCPV(catpkg)
987 pinfo.latest_upstream_cpv = self._FindUpstreamCPV(catpkg,
990 # The upstream version can be either latest stable or latest overall,
991 # or specified explicitly by the user at the command line. In the latter
992 # case, 'upstream_cpv' will already be set.
993 if not pinfo.upstream_cpv:
994 if not self._unstable_ok:
995 pinfo.upstream_cpv = pinfo.stable_upstream_cpv
997 pinfo.upstream_cpv = pinfo.latest_upstream_cpv
999 # Perform the actual upgrade, if requested.
1000 pinfo.cpv_cmp_upstream = None
1001 pinfo.upgraded_cpv = None
1002 if pinfo.upstream_cpv:
1003 # cpv_cmp_upstream values: 0 = current, >0 = outdated, <0 = futuristic!
1004 pinfo.cpv_cmp_upstream = Upgrader._CmpCpv(pinfo.upstream_cpv, cpv)
1006 # Determine whether upgrade of this package is requested.
1007 if self._PkgUpgradeRequested(pinfo):
1008 if self._PkgUpgradeStaged(pinfo.upstream_cpv):
1009 oper.Notice('Determined that %s is already staged.' %
1011 pinfo.upgraded_cpv = pinfo.upstream_cpv
1012 elif pinfo.cpv_cmp_upstream > 0:
1013 pinfo.upgraded_cpv = self._CopyUpstreamPackage(pinfo.upstream_cpv)
1014 elif pinfo.cpv_cmp_upstream == 0:
1016 oper.Notice('Forcing upgrade of existing %s.' %
1018 upgraded_cpv = self._CopyUpstreamPackage(pinfo.upstream_cpv)
1019 pinfo.upgraded_cpv = upgraded_cpv
1021 oper.Warning('Not upgrading %s; it already exists in source.\n'
1022 'To force upgrade of this version specify --force.' %
1024 elif self._PkgUpgradeRequested(pinfo):
1025 raise RuntimeError('Unable to find upstream package for upgrading %s.' %
1028 if pinfo.upgraded_cpv:
1029 # Deal with keywords now. We always run this logic as we sometimes will
1030 # stabilizing keywords other than just our own (the unsupported arches).
1031 self._SetUpgradedMaskBits(pinfo)
1032 ebuild_path = Upgrader._GetEbuildPathFromCpv(pinfo.upgraded_cpv)
1033 self._StabilizeEbuild(os.path.join(self._stable_repo, ebuild_path))
1035 # Add all new package files to git.
1036 self._RunGit(self._stable_repo, ['add', pinfo.package])
1038 # Update profiles/categories.
1039 self._UpdateCategories(pinfo)
1041 # Regenerate the cache. In theory, this might glob too much, but
1042 # in practice, this should be fine for now ...
1043 cache_files = 'metadata/md5-cache/%s-[0-9]*' % pinfo.package
1044 self._RunGit(self._stable_repo, ['rm', '--ignore-unmatch', '-q', '-f',
1046 cmd = ['egencache', '--update', '--repo=portage-stable', pinfo.package]
1047 egen_result = cros_build_lib.RunCommand(cmd, print_cmd=False,
1048 redirect_stdout=True,
1049 combine_stdout_stderr=True)
1050 if egen_result.returncode != 0:
1051 raise RuntimeError('Failed to regenerate md5-cache for %r.\n'
1052 'Output of %r:\n%s' %
1053 (pinfo.package, ' '.join(cmd), egen_result.output))
1055 self._RunGit(self._stable_repo, ['add', cache_files])
1057 return bool(pinfo.upgraded_cpv)
1059 def _UpdateCategories(self, pinfo):
1060 """Update profiles/categories to include category in |pinfo|, if needed."""
1062 if pinfo.category not in self._stable_repo_categories:
1063 self._stable_repo_categories.add(pinfo.category)
1064 self._WriteStableRepoCategories()
1066 def _VerifyPackageUpgrade(self, pinfo):
1067 """Verify that the upgraded package in |pinfo| passes checks."""
1068 self._VerifyEbuildOverlay(pinfo.upgraded_cpv, self.STABLE_OVERLAY_NAME,
1069 pinfo.cpv_cmp_upstream == 0)
1071 def _PackageReport(self, pinfo):
1072 """Report on whatever was done with package in |pinfo|."""
1074 pinfo.state = self._GetPackageUpgradeState(pinfo)
1077 # Print a quick summary of package status.
1078 self._PrintPackageLine(pinfo)
1080 # Add a row to status table for this package
1081 self._AppendPackageRow(pinfo)
1083 def _ExtractUpgradedPkgs(self, upgrade_lines):
1084 """Extracts list of packages from standard commit |upgrade_lines|."""
1085 # Expecting message lines like this (return just package names):
1086 # Upgraded sys-libs/ncurses to version 5.7-r7 on amd64, arm, x86
1087 # Upgraded sys-apps/less to version 441 on amd64, arm
1088 # Upgraded sys-apps/less to version 442 on x86
1090 regexp = re.compile(r'^%s\s+\S+/(\S+)\s' % UPGRADED)
1091 for line in upgrade_lines:
1092 match = regexp.search(line)
1094 pkgs.add(match.group(1))
1098 def _CreateCommitMessage(self, upgrade_lines, remaining_lines=None):
1099 """Create appropriate git commit message for upgrades in |upgrade_lines|."""
1101 upgrade_pkgs = self._ExtractUpgradedPkgs(upgrade_lines)
1102 upgrade_count = len(upgrade_pkgs)
1103 upgrade_str = '\n'.join(upgrade_lines)
1104 if upgrade_count < 6:
1105 message = ('%s: upgraded package%s to upstream' %
1106 (', '.join(upgrade_pkgs), '' if upgrade_count == 1 else 's'))
1108 message = 'Upgraded the following %d packages' % upgrade_count
1109 message += '\n\n' + upgrade_str + '\n'
1112 # Keep previous remaining lines verbatim.
1113 message += '\n%s\n' % '\n'.join(remaining_lines)
1115 # The space before <fill-in> (at least for TEST=) fails pre-submit check,
1116 # which is the intention here.
1117 message += '\nBUG= <fill-in>'
1118 message += '\nTEST= <fill-in>'
1122 def _AmendCommitMessage(self, upgrade_lines):
1123 """Create git commit message combining |upgrade_lines| with last commit."""
1124 # First get the body of the last commit message.
1125 git_cmd = ['show', '-s', '--format=%b']
1126 result = self._RunGit(self._stable_repo, git_cmd, redirect_stdout=True)
1127 body = result.output
1129 remaining_lines = []
1130 # Extract the upgrade_lines of last commit. Everything after the
1131 # empty line is preserved verbatim.
1132 # Expecting message body like this:
1133 # Upgraded sys-libs/ncurses to version 5.7-r7 on amd64, arm, x86
1134 # Upgraded sys-apps/less to version 441 on amd64, arm, x86
1136 # BUG=chromium-os:20923
1137 # TEST=trybot run of chromiumos-sdk
1139 for line in body.split('\n'):
1140 if not before_break:
1141 remaining_lines.append(line)
1143 if re.search(r'^%s\s+' % UPGRADED, line):
1144 upgrade_lines.append(line)
1146 # If the lines in the message body are not in the expected
1147 # format simply push them to the end of the new commit
1148 # message body, but left intact.
1149 oper.Warning('It looks like the existing commit message '
1150 'that you are amending was not generated by\n'
1151 'this utility. Appending previous commit '
1152 'message to newly generated message.')
1153 before_break = False
1154 remaining_lines.append(line)
1156 before_break = False
1158 return self._CreateCommitMessage(upgrade_lines, remaining_lines)
1160 def _GiveEmergeResults(self, pinfolist):
1161 """Summarize emerge checks, raise RuntimeError if there is a problem."""
1163 upgraded_pinfos = [pinfo for pinfo in pinfolist if pinfo.upgraded_cpv]
1164 upgraded_cpvs = [pinfo.upgraded_cpv for pinfo in upgraded_pinfos]
1165 masked_cpvs = set([pinfo.upgraded_cpv for pinfo in upgraded_pinfos
1166 if not pinfo.upgraded_unmasked])
1168 (ok, cmd, output) = self._AreEmergeable(upgraded_cpvs)
1171 # If any of the upgraded_cpvs are masked, then emerge should have
1172 # failed. Give a helpful message. If it didn't fail then panic.
1174 raise RuntimeError('Emerge passed for masked package(s)! Something '
1175 'fishy here. Emerge output follows:\n%s\n'
1176 'Show this to the build team.' % output)
1179 oper.Error('\nEmerge output for "%s" on %s follows:' %
1180 (cmd, self._curr_arch))
1182 for masked_cpv in masked_cpvs:
1183 self._GiveMaskedError(masked_cpv, output)
1184 raise RuntimeError('\nOne or more upgraded packages are masked '
1188 oper.Notice('Confirmed that all upgraded packages can be emerged '
1189 'on %s after upgrade.' % self._curr_board)
1191 oper.Error('Packages cannot be emerged after upgrade. The output '
1192 'of "%s" follows:' % cmd)
1194 raise RuntimeError('Failed to complete upgrades on %s (see above). '
1195 'Address the emerge errors before continuing.' %
1198 def _UpgradePackages(self, pinfolist):
1199 """Given a list of cpv pinfos, adds the upstream cpv to the pinfos."""
1200 self._curr_table.Clear()
1203 upgrades_this_run = False
1204 for pinfo in pinfolist:
1205 if self._UpgradePackage(pinfo):
1206 self._upgrade_cnt += 1
1207 upgrades_this_run = True
1209 # The verification of upgrades needs to happen after upgrades are done.
1210 # The reason is that it cannot be guaranteed that pinfolist is ordered
1211 # such that dependencies are satisified after each individual upgrade,
1212 # because one or more of the packages may only exist upstream.
1213 for pinfo in pinfolist:
1214 if pinfo.upgraded_cpv:
1215 self._VerifyPackageUpgrade(pinfo)
1217 self._PackageReport(pinfo)
1219 if upgrades_this_run:
1220 self._GiveEmergeResults(pinfolist)
1222 if self._IsInUpgradeMode():
1223 # If there were any ebuilds staged before running this script, then
1224 # make sure they were targeted in pinfolist. If not, abort.
1225 self._CheckStagedUpgrades(pinfolist)
1226 except RuntimeError as ex:
1229 raise RuntimeError('\nTo reset all changes in %s now:\n'
1230 ' cd %s; git reset --hard; cd -' %
1231 (self._stable_repo, self._stable_repo))
1232 # Allow the changes to stay staged so that the user can attempt to address
1233 # the issue (perhaps an edit to package.mask is required, or another
1234 # package must also be upgraded).
1236 def _CheckStagedUpgrades(self, pinfolist):
1237 """Raise RuntimeError if staged upgrades are not also in |pinfolist|."""
1238 # This deals with the situation where a previous upgrade run staged one or
1239 # more package upgrades, but did not commit them because it found an error
1240 # of some kind. This is ok, as long as subsequent runs continue to request
1241 # an upgrade of that package again (presumably with the problem fixed).
1242 # However, if a subsequent run does not mention that package then it should
1243 # abort. The user must reset those staged changes first.
1245 if self._stable_repo_status:
1248 # Go over files with pre-existing git statuses.
1249 filelist = self._stable_repo_status.keys()
1250 ebuilds = [e for e in filelist if e.endswith('.ebuild')]
1252 for ebuild in ebuilds:
1253 status = self._stable_repo_status[ebuild]
1254 (_overlay, cat, pn, _pv) = self._SplitEBuildPath(ebuild)
1255 package = '%s/%s' % (cat, pn)
1257 # As long as this package is involved in an upgrade this is fine.
1258 matching_pinfos = [pi for pi in pinfolist if pi.package == package]
1259 if not matching_pinfos:
1260 err_msgs.append('Staged %s (status=%s) is not an upgrade target.' %
1264 raise RuntimeError('%s\n'
1265 'Add to upgrade targets or reset staged changes.' %
1266 '\n'.join(err_msgs))
1268 def _GenParallelEmergeArgv(self, args):
1269 """Creates an argv for parallel_emerge using current options and |args|."""
1270 argv = ['--emptytree', '--pretend']
1271 if self._curr_board and self._curr_board != self.HOST_BOARD:
1272 argv.append('--board=%s' % self._curr_board)
1273 if not self._verbose:
1274 argv.append('--quiet')
1276 argv.append('--root-deps=rdeps')
1281 def _SetPortTree(self, settings, trees):
1282 """Set self._porttree from portage |settings| and |trees|."""
1283 root = settings['ROOT']
1284 self._porttree = trees[root]['porttree']
1286 def _GetPortageDBAPI(self):
1287 """Retrieve the Portage dbapi object, if available."""
1289 return self._porttree.dbapi
1290 except AttributeError:
1293 def _CreatePInfoFromCPV(self, cpv, cpv_key=None):
1294 """Return a basic pinfo object created from |cpv|."""
1296 self._FillPInfoFromCPV(pinfo, cpv, cpv_key)
1299 def _FillPInfoFromCPV(self, pinfo, cpv, cpv_key=None):
1300 """Flesh out |pinfo| from |cpv|."""
1301 pkg = Upgrader._GetCatPkgFromCpv(cpv)
1302 (cat, pn) = pkg.split('/')
1305 pinfo.upstream_cpv = None
1308 pinfo.package_name = pn
1309 pinfo.category = cat
1312 setattr(pinfo, cpv_key, cpv)
1314 def _GetCurrentVersions(self, target_pinfolist):
1315 """Returns a list of pkg pinfos of the current package dependencies.
1317 The dependencies are taken from giving the 'package' values in each
1318 pinfo of |target_pinfolist| to (parallel_)emerge.
1320 The returned list is ordered such that the dependencies of any mentioned
1321 package occur earlier in the list.
1324 for pinfo in target_pinfolist:
1325 local_cpv = pinfo.cpv
1326 if local_cpv and local_cpv != WORLD_TARGET:
1327 emerge_args.append('=' + local_cpv)
1329 emerge_args.append(pinfo.package)
1330 argv = self._GenParallelEmergeArgv(emerge_args)
1332 deps = parallel_emerge.DepGraphGenerator()
1333 deps.Initialize(argv)
1336 deps_tree, deps_info = deps.GenDependencyTree()
1338 oper.Error('Run of parallel_emerge exited with error while assembling'
1339 ' package dependencies (error message should be above).\n'
1340 'Command effectively was:\n%s' %
1341 ' '.join(['parallel_emerge'] + argv))
1342 oper.Error('Address the source of the error, then run again.')
1344 self._SetPortTree(deps.emerge.settings, deps.emerge.trees)
1345 self._deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
1347 cpvlist = Upgrader._GetPreOrderDepGraph(self._deps_graph)
1352 # See if this cpv was in target_pinfolist
1354 for pinfo in target_pinfolist:
1355 if cpv == pinfo.cpv:
1356 pinfolist.append(pinfo)
1360 pinfolist.append(self._CreatePInfoFromCPV(cpv, cpv_key='cpv'))
1364 def _FinalizeLocalPInfolist(self, orig_pinfolist):
1365 """Filters and fleshes out |orig_pinfolist|, returns new list.
1367 Each pinfo object is assumed to have entries for:
1368 cpv, package, package_name, category
1371 for pinfo in orig_pinfolist:
1372 # No need to report or try to upgrade chromeos-base packages.
1373 if pinfo.category == 'chromeos-base':
1376 dbapi = self._GetPortageDBAPI()
1377 ebuild_path = dbapi.findname2(pinfo.cpv)[0]
1379 # This has only happened once. See crosbug.com/26385.
1380 # In that case, this meant the package, while in the deps graph,
1381 # was actually to be uninstalled. How is that possible? The
1382 # package was newly added to package.provided. So skip it.
1383 oper.Notice('Skipping %r from deps graph, as it appears to be'
1384 ' scheduled for uninstall.' % pinfo.cpv)
1387 (overlay, _cat, pn, pv) = self._SplitEBuildPath(ebuild_path)
1388 ver_rev = pv.replace(pn + '-', '')
1389 slot, = dbapi.aux_get(pinfo.cpv, ['SLOT'])
1392 pinfo.overlay = overlay
1393 pinfo.version_rev = ver_rev
1394 pinfo.package_ver = pv
1396 pinfolist.append(pinfo)
1400 # TODO(mtennant): It is likely this method can be yanked now that all
1401 # attributes in PInfo are initialized to something (None).
1402 # TODO(mtennant): This should probably not return anything, since it
1403 # also modifies the list that is passed in.
1404 def _FinalizeUpstreamPInfolist(self, pinfolist):
1405 """Adds missing values in upstream |pinfolist|, returns list."""
1407 for pinfo in pinfolist:
1408 pinfo.slot = NOT_APPLICABLE
1409 pinfo.overlay = NOT_APPLICABLE
1410 pinfo.version_rev = NOT_APPLICABLE
1411 pinfo.package_ver = NOT_APPLICABLE
1415 def _ResolveAndVerifyArgs(self, args, upgrade_mode):
1416 """Resolve |args| to full pkgs, and check validity of each.
1418 Each argument will be resolved to a full category/packagename, if possible,
1419 by looking in both the local overlays and the upstream overlay. Any
1420 argument that cannot be resolved will raise a RuntimeError.
1422 Arguments that specify a specific version of a package are only
1423 allowed when |upgrade_mode| is True.
1425 The 'world' target is handled as a local package.
1427 Any errors will raise a RuntimeError.
1429 Return list of package pinfos, one for each argument. Each will have:
1430 'user_arg' = Original command line argument package was resolved from
1431 'package' = Resolved category/package_name
1432 'package_name' = package_name
1433 'category' = category (None for 'world' target)
1434 Packages found in local overlays will also have:
1435 'cpv' = Current cpv ('world' for 'world' target)
1436 Packages found upstream will also have:
1437 'upstream_cpv' = Upstream cpv
1442 pinfo = PInfo(user_arg=arg)
1444 if arg == WORLD_TARGET:
1445 # The 'world' target is a special case. Consider it a valid target
1446 # locally, but not an upstream package.
1448 pinfo.package_name = arg
1449 pinfo.category = None
1452 catpkg = Upgrader._GetCatPkgFromCpv(arg)
1453 verrev = Upgrader._GetVerRevFromCpv(arg)
1455 if verrev and not upgrade_mode:
1456 raise RuntimeError('Specifying specific versions is only allowed '
1457 'in upgrade mode. Do not know what to do with '
1460 # Local cpv search ignores version in argument, if any. If version is
1461 # in argument, though, it *must* be found upstream.
1462 local_arg = catpkg if catpkg else arg
1464 local_cpv = self._FindCurrentCPV(local_arg)
1465 upstream_cpv = self._FindUpstreamCPV(arg, self._unstable_ok)
1467 # Old-style virtual packages will resolve to their target packages,
1468 # which we do not want here because if the package 'virtual/foo' was
1469 # specified at the command line we want to try upgrading the actual
1470 # 'virtual/foo' package, not whatever package equery resolves it to.
1471 # This only matters when 'virtual/foo' is currently an old-style
1472 # virtual but a new-style virtual for it exists upstream which we
1473 # want to upgrade to. For new-style virtuals, equery will resolve
1474 # 'virtual/foo' to 'virtual/foo', which is fine.
1475 if arg.startswith('virtual/'):
1476 if local_cpv and not local_cpv.startswith('virtual/'):
1479 if not upstream_cpv and upgrade_mode:
1480 # See if --unstable-ok is required for this upstream version.
1481 if not self._unstable_ok and self._FindUpstreamCPV(arg, True):
1482 raise RuntimeError('Upstream "%s" is unstable on %s. Re-run with '
1483 '--unstable-ok option?' % (arg, self._curr_arch))
1485 raise RuntimeError('Unable to find "%s" upstream on %s.' %
1486 (arg, self._curr_arch))
1488 any_cpv = local_cpv if local_cpv else upstream_cpv
1490 msg = ('Unable to resolve "%s" as a package either local or upstream.'
1492 if arg.find('/') < 0:
1493 msg = msg + ' Try specifying the full category/package_name.'
1495 raise RuntimeError(msg)
1497 self._FillPInfoFromCPV(pinfo, any_cpv)
1498 pinfo.cpv = local_cpv
1499 pinfo.upstream_cpv = upstream_cpv
1500 if local_cpv and upstream_cpv:
1501 oper.Notice('Resolved "%s" to "%s" (local) and "%s" (upstream).' %
1502 (arg, local_cpv, upstream_cpv))
1504 oper.Notice('Resolved "%s" to "%s" (local).' %
1507 oper.Notice('Resolved "%s" to "%s" (upstream).' %
1508 (arg, upstream_cpv))
1510 pinfolist.append(pinfo)
1514 def PrepareToRun(self):
1515 """Checkout upstream gentoo if necessary, and any other prep steps."""
1517 if not os.path.exists(os.path.join(
1518 self._upstream, '.git', 'shallow')):
1519 osutils.RmDir(self._upstream, ignore_missing=True)
1521 if os.path.exists(self._upstream):
1522 if self._local_only:
1523 oper.Notice('Using upstream cache as-is (no network) %s.' %
1526 # Recheck the pathway; it's possible in switching off alternates,
1527 # this was converted down to a depth=1 repo.
1529 oper.Notice('Updating previously created upstream cache at %s.' %
1531 self._RunGit(self._upstream, ['remote', 'set-url', 'origin',
1532 self.PORTAGE_GIT_URL])
1533 self._RunGit(self._upstream, ['remote', 'update'])
1534 self._RunGit(self._upstream, ['checkout', self.ORIGIN_GENTOO],
1535 redirect_stdout=True, combine_stdout_stderr=True)
1537 if self._local_only:
1538 oper.Die('--local-only specified, but no local cache exists. '
1539 'Re-run w/out --local-only to create cache automatically.')
1541 root = os.path.dirname(self._upstream)
1542 osutils.SafeMakedirs(root)
1543 # Create local copy of upstream gentoo.
1544 oper.Notice('Cloning origin/gentoo at %s as upstream reference.' %
1546 name = os.path.basename(self._upstream)
1547 args = ['clone', '--branch', os.path.basename(self.ORIGIN_GENTOO)]
1548 args += ['--depth', '1', self.PORTAGE_GIT_URL, name]
1549 self._RunGit(root, args)
1551 # Create a README file to explain its presence.
1552 with open(self._upstream + '-README', 'w') as f:
1553 f.write('Directory at %s is local copy of upstream '
1554 'Gentoo/Portage packages. Used by cros_portage_upgrade.\n'
1555 'Feel free to delete if you want the space back.\n' %
1558 # An empty directory is needed to trick equery later.
1559 self._emptydir = tempfile.mkdtemp()
1561 def RunCompleted(self):
1562 """Undo any checkout of upstream gentoo if requested."""
1563 if self._no_upstream_cache:
1564 oper.Notice('Removing upstream cache at %s as requested.'
1566 osutils.RmDir(self._upstream, ignore_missing=True)
1568 # Remove the README file, too.
1569 readmepath = self._upstream + '-README'
1570 osutils.SafeUnlink(readmepath)
1572 oper.Notice('Keeping upstream cache at %s.' % self._upstream)
1575 osutils.RmDir(self._emptydir, ignore_missing=True)
1577 def CommitIsStaged(self):
1578 """Return True if upgrades are staged and ready for a commit."""
1579 return bool(self._upgrade_cnt)
1582 """Commit whatever has been prepared in the stable repo."""
1583 # Trying to create commit message body lines that look like these:
1584 # Upgraded foo/bar-1.2.3 to version 1.2.4 on x86
1585 # Upgraded foo/baz to version 2 on arm AND version 3 on amd64, x86
1587 commit_lines = [] # Lines for the body of the commit message
1588 pkg_overlays = {} # Overlays for upgraded packages in non-portage overlays.
1590 # Assemble hash of COL_UPGRADED column names by arch.
1592 for arch in self._master_archs:
1593 tmp_col = utable.UpgradeTable.COL_UPGRADED
1594 col = utable.UpgradeTable.GetColumnName(tmp_col, arch)
1595 upgraded_cols[arch] = col
1597 table = self._master_table
1599 pkg = row[table.COL_PACKAGE]
1600 pkg_commit_line = None
1602 # First determine how many unique upgraded versions there are.
1603 upgraded_versarch = {}
1604 for arch in self._master_archs:
1605 upgraded_ver = row[upgraded_cols[arch]]
1607 # This package has been upgraded for this arch.
1608 upgraded_versarch.setdefault(upgraded_ver, []).append(arch)
1610 # Save the overlay this package is originally from, if the overlay
1611 # is not a Portage overlay (e.g. chromiumos-overlay).
1612 ovrly_col = utable.UpgradeTable.COL_OVERLAY
1613 ovrly_col = utable.UpgradeTable.GetColumnName(ovrly_col, arch)
1614 ovrly = row[ovrly_col]
1615 if (ovrly != NOT_APPLICABLE and
1616 ovrly != self.UPSTREAM_OVERLAY_NAME and
1617 ovrly != self.STABLE_OVERLAY_NAME):
1618 pkg_overlays[pkg] = ovrly
1620 if upgraded_versarch:
1621 pkg_commit_line = '%s %s to ' % (UPGRADED, pkg)
1622 pkg_commit_line += ' AND '.join(
1623 'version %s on %s' % (upgraded_ver, ', '.join(sorted(archlist)))
1624 for upgraded_ver, archlist in upgraded_versarch.iteritems())
1625 commit_lines.append(pkg_commit_line)
1629 message = self._AmendCommitMessage(commit_lines)
1630 self._RunGit(self._stable_repo, ['commit', '--amend', '-m', message])
1632 message = self._CreateCommitMessage(commit_lines)
1633 self._RunGit(self._stable_repo, ['commit', '-m', message])
1636 'Upgrade changes committed (see above),'
1637 ' but message needs edit BY YOU:\n'
1638 ' cd %s; git commit --amend; cd -' %
1640 # See if any upgraded packages are in non-portage overlays now, meaning
1641 # they probably require a patch and should not go into portage-stable.
1643 lines = ['%s [%s]' % (p, pkg_overlays[p]) for p in pkg_overlays]
1645 'The following packages were coming from a non-portage'
1646 ' overlay, which means they were probably patched.\n'
1647 'You should consider whether the upgraded package'
1648 ' needs the same patches applied now.\n'
1649 'If so, do not commit these changes in portage-stable.'
1650 ' Instead, copy them to the applicable overlay dir.\n'
1654 'To remove any individual file above from commit do:\n'
1655 ' cd %s; git reset HEAD~ <filepath>; rm <filepath>;'
1656 ' git commit --amend -C HEAD; cd -' %
1660 'If you wish to undo all the changes to %s:\n'
1661 ' cd %s; git reset --hard HEAD~; cd -' %
1662 (self.STABLE_OVERLAY_NAME, self._stable_repo))
1664 def PreRunChecks(self):
1665 """Run any board-independent validation checks before Run is called."""
1666 # Upfront check(s) if upgrade is requested.
1667 if self._upgrade or self._upgrade_deep:
1668 # Stable source must be on branch.
1669 self._CheckStableRepoOnBranch()
1671 def CheckBoardList(self, boards):
1672 """Validate list of specified |boards| before running any of them."""
1674 # If this is an upgrade run (i.e. --upgrade was specified), then in
1675 # almost all cases we want all our supported architectures to be covered.
1676 if self._IsInUpgradeMode():
1678 for board in boards:
1679 board_archs.add(Upgrader._FindBoardArch(board))
1681 if not STANDARD_BOARD_ARCHS.issubset(board_archs):
1682 # Only proceed if user acknowledges.
1683 oper.Warning('You have selected boards for archs %r, which does not'
1684 ' cover all standard archs %r' %
1685 (sorted(board_archs), sorted(STANDARD_BOARD_ARCHS)))
1686 oper.Warning('If you continue with this upgrade you may break'
1687 ' builds for architectures not covered by your\n'
1688 'boards. Continue only if you have a reason to limit'
1689 ' this upgrade to these specific architectures.\n')
1690 if not cros_build_lib.BooleanPrompt(
1691 prompt="Do you want to continue anyway?", default=False):
1692 raise RuntimeError('Missing one or more of the standard archs')
1694 def RunBoard(self, board):
1695 """Runs the upgrader based on the supplied options and arguments.
1697 Currently just lists all package dependencies in pre-order along with
1700 # Preserve status report for entire stable repo (output of 'git status -s').
1701 self._SaveStatusOnStableRepo()
1702 # Read contents of profiles/categories for later checks
1703 self._LoadStableRepoCategories()
1705 self._porttree = None
1706 self._deps_graph = None
1708 self._curr_board = board
1709 self._curr_arch = Upgrader._FindBoardArch(board)
1710 upgrade_mode = self._IsInUpgradeMode()
1711 self._curr_table = utable.UpgradeTable(self._curr_arch,
1712 upgrade=upgrade_mode,
1715 if self._AnyChangesStaged():
1716 self._StashChanges()
1719 target_pinfolist = self._ResolveAndVerifyArgs(self._args, upgrade_mode)
1720 upstream_only_pinfolist = [pi for pi in target_pinfolist if not pi.cpv]
1721 if not upgrade_mode and upstream_only_pinfolist:
1722 # This means that not all arguments were found in local source, which is
1723 # only allowed in upgrade mode.
1724 msg = ('The following packages were not found in current overlays'
1725 ' (but they do exist upstream):\n%s' %
1726 '\n'.join([pinfo.user_arg for pinfo in upstream_only_pinfolist]))
1727 raise RuntimeError(msg)
1729 full_pinfolist = None
1732 # Shallow upgrade mode only cares about targets as they were
1734 full_pinfolist = self._FinalizeUpstreamPInfolist(target_pinfolist)
1736 # Assembling dependencies only matters in status report mode or
1737 # if --upgrade-deep was requested.
1738 local_target_pinfolist = [pi for pi in target_pinfolist if pi.cpv]
1739 if local_target_pinfolist:
1740 oper.Notice('Assembling package dependencies.')
1741 full_pinfolist = self._GetCurrentVersions(local_target_pinfolist)
1742 full_pinfolist = self._FinalizeLocalPInfolist(full_pinfolist)
1746 # Append any command line targets that were not found in current
1747 # overlays. The idea is that they will still be found upstream
1750 tmp_list = self._FinalizeUpstreamPInfolist(upstream_only_pinfolist)
1751 full_pinfolist = full_pinfolist + tmp_list
1753 self._UnstashAnyChanges()
1754 self._UpgradePackages(full_pinfolist)
1757 self._DropAnyStashedChanges()
1759 # Merge tables together after each run.
1760 self._master_cnt += 1
1761 self._master_archs.add(self._curr_arch)
1762 if self._master_table:
1763 tables = [self._master_table, self._curr_table]
1764 self._master_table = mps.MergeTables(tables)
1766 self._master_table = self._curr_table
1767 self._master_table._arch = None
1769 def WriteTableFiles(self, csv=None):
1770 """Write |self._master_table| to |csv| file, if requested."""
1772 # Sort the table by package name, then slot
1773 def PkgSlotSort(row):
1774 return (row[self._master_table.COL_PACKAGE],
1775 row[self._master_table.COL_SLOT])
1776 self._master_table.Sort(PkgSlotSort)
1779 filehandle = open(csv, 'w')
1780 oper.Notice('Writing package status as csv to %s.' % csv)
1781 self._master_table.WriteCSV(filehandle)
1783 elif not self._IsInUpgradeMode():
1784 oper.Notice('Package status report file not requested (--to-csv).')
1786 def SayGoodbye(self):
1787 """Print any final messages to user."""
1788 if not self._IsInUpgradeMode():
1789 # Without this message users are confused why running a script
1790 # with 'upgrade' in the name does not actually do an upgrade.
1791 oper.Warning('Completed status report run. To run in "upgrade"'
1792 ' mode include the --upgrade option.')
1794 def _BoardIsSetUp(board):
1795 """Return true if |board| has been setup."""
1796 return os.path.isdir(cros_build_lib.GetSysroot(board=board))
1798 def _CreateParser():
1799 """Create the optparser.parser object for command-line args."""
1801 'There are essentially two "modes": status report mode and '
1802 'upgrade mode.\nStatus report mode is the default; upgrade '
1803 'mode is enabled by either --upgrade or --upgrade-deep.\n'
1805 'In either mode, packages can be specified in any manner '
1806 'commonly accepted by Portage tools. For example:\n'
1807 ' category/package_name\n'
1809 ' category/package_name-version (upgrade mode only)\n'
1811 'Status report mode will report on the status of the specified '
1812 'packages relative to upstream,\nwithout making any changes. '
1813 'In this mode, the specified packages are often high-level\n'
1814 'targets such as "virtual/target-os". '
1815 'The --to-csv option is often used in this mode.\n'
1816 'The --unstable-ok option in this mode will make '
1817 'the upstream comparison (e.g. "needs update") be\n'
1818 'relative to the latest upstream version, stable or not.\n'
1820 'Upgrade mode will attempt to upgrade the specified '
1821 'packages to one of the following versions:\n'
1822 '1) The version specified in argument (e.g. foo/bar-1.2.3)\n'
1823 '2) The latest stable version upstream (the default)\n'
1824 '3) The latest overall version upstream (with --unstable-ok)\n'
1826 'Unlike with --upgrade, if --upgrade-deep is specified, '
1827 'then the package dependencies will also be upgraded.\n'
1828 'In upgrade mode, it is ok if the specified packages only '
1830 'The --force option can be used to do a package upgrade '
1831 'even if the local version matches the upstream version.\n'
1833 'Status report mode examples:\n'
1834 '> cros_portage_upgrade --board=arm-generic:x86-generic '
1835 '--to-csv=cros-aebl.csv virtual/target-os\n'
1836 '> cros_portage_upgrade --unstable-ok --board=x86-mario '
1837 '--to-csv=cros_test-mario virtual/target-os virtual/target-os-dev '
1838 'virtual/target-os-test\n'
1839 'Upgrade mode examples:\n'
1840 '> cros_portage_upgrade --board=arm-generic:x86-generic '
1841 '--upgrade sys-devel/gdb virtual/yacc\n'
1842 '> cros_portage_upgrade --unstable-ok --board=x86-mario '
1843 '--upgrade-deep gdata\n'
1844 '> cros_portage_upgrade --board=x86-generic --upgrade '
1845 'media-libs/libpng-1.2.45\n'
1849 parser = commandline.ArgumentParser(epilog=epilog)
1850 parser.add_argument('packages', nargs='*', default=None,
1851 help='Packages to process.')
1852 parser.add_argument('--amend', action='store_true', default=False,
1853 help='Amend existing commit when doing upgrade.')
1854 parser.add_argument('--board', default=None,
1855 help='Target board(s), colon-separated')
1856 parser.add_argument('--force', action='store_true', default=False,
1857 help='Force upgrade even if version already in source')
1858 parser.add_argument('--host', action='store_true', default=False,
1859 help='Host target pseudo-board')
1860 parser.add_argument('--no-upstream-cache', action='store_true', default=False,
1861 help='Do not preserve cached upstream for future runs')
1862 parser.add_argument('--rdeps', action='store_true', default=False,
1863 help='Use runtime dependencies only')
1864 parser.add_argument('--srcroot', type='path',
1865 default='%s/trunk/src' % os.environ['HOME'],
1866 help='Path to root src directory [default: %(default)s]')
1867 parser.add_argument('--to-csv', dest='csv_file', type='path',
1868 default=None, help='File to store csv-formatted results')
1869 parser.add_argument('--upgrade', action='store_true', default=False,
1870 help='Upgrade target package(s) only')
1871 parser.add_argument('--upgrade-deep', action='store_true', default=False,
1872 help='Upgrade target package(s) and all dependencies')
1873 parser.add_argument('--upstream', type='path',
1874 default=Upgrader.UPSTREAM_TMP_REPO,
1875 help='Latest upstream repo location '
1876 '[default: %(default)s]')
1877 parser.add_argument('--unstable-ok', action='store_true', default=False,
1878 help='Use latest upstream ebuild, stable or not')
1879 parser.add_argument('--verbose', action='store_true', default=False,
1880 help='Enable verbose output (for debugging)')
1881 parser.add_argument('-l', '--local-only', action='store_true', default=False,
1882 help='Do not attempt to update local portage cache')
1887 """Main function."""
1888 parser = _CreateParser()
1889 options = parser.parse_args(argv)
1890 # TODO: Can't freeze until options.host modification below is sorted.
1893 oper.verbose = options.verbose
1896 # Do some argument checking.
1899 if not options.board and not options.host:
1900 parser.print_usage()
1901 oper.Die('Board (or host) is required.')
1903 if not options.packages:
1904 parser.print_usage()
1905 oper.Die('No packages provided.')
1907 # The --upgrade and --upgrade-deep options are mutually exclusive.
1908 if options.upgrade_deep and options.upgrade:
1909 parser.print_usage()
1910 oper.Die('The --upgrade and --upgrade-deep options '
1911 'are mutually exclusive.')
1913 # The --force option only makes sense with --upgrade or --upgrade-deep.
1914 if options.force and not (options.upgrade or options.upgrade_deep):
1915 parser.print_usage()
1916 oper.Die('The --force option requires --upgrade or --upgrade-deep.')
1918 # If --to-csv given verify file can be opened for write.
1919 if options.csv_file:
1921 osutils.WriteFile(options.csv_file, '')
1922 except IOError as ex:
1923 parser.print_usage()
1924 oper.Die('Unable to open %s for writing: %s' % (options.csv_file,
1927 upgrader = Upgrader(options)
1928 upgrader.PreRunChecks()
1930 # Automatically handle board 'host' as 'amd64-host'.
1933 boards = options.board.split(':')
1935 # Specifying --board=host is equivalent to --host.
1936 if 'host' in boards:
1939 boards = [b for b in boards if b != 'host']
1941 # Make sure host pseudo-board is run first.
1942 if options.host and Upgrader.HOST_BOARD not in boards:
1943 boards.insert(0, Upgrader.HOST_BOARD)
1944 elif Upgrader.HOST_BOARD in boards:
1945 boards = [b for b in boards if b != Upgrader.HOST_BOARD]
1946 boards.insert(0, Upgrader.HOST_BOARD)
1948 # Check that all boards have been setup first.
1949 for board in boards:
1950 if (board != Upgrader.HOST_BOARD and not _BoardIsSetUp(board)):
1951 parser.print_usage()
1952 oper.Die('You must setup the %s board first.' % board)
1954 # If --board and --upgrade are given then in almost all cases
1955 # the user should cover all architectures.
1957 non_host_boards = [b for b in boards if b != Upgrader.HOST_BOARD]
1958 upgrader.CheckBoardList(non_host_boards)
1962 upgrader.PrepareToRun()
1964 for board in boards:
1965 oper.Notice('Running with board %s.' % board)
1966 upgrader.RunBoard(board)
1967 except RuntimeError as ex:
1972 upgrader.RunCompleted()
1975 oper.Die('Failed with above errors.')
1977 if upgrader.CommitIsStaged():
1980 # TODO(mtennant): Move stdout output to here, rather than as-we-go. That
1981 # way it won't come out for each board. Base it on contents of final table.
1982 # Make verbose-dependent?
1984 upgrader.WriteTableFiles(csv=options.csv_file)
1986 upgrader.SayGoodbye()