src/third_party/chromite/cbuildbot/stages/build_stages.py

   1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Module containing the build stages."""
   6
   7 import functools
   8 import glob
   9 import os
  10
  11 from chromite.cbuildbot import commands
  12 from chromite.cbuildbot import constants
  13 from chromite.cbuildbot import failures_lib
  14 from chromite.cbuildbot import repository
  15 from chromite.cbuildbot.stages import generic_stages
  16 from chromite.cbuildbot.stages import test_stages
  17 from chromite.lib import cros_build_lib
  18 from chromite.lib import git
  19 from chromite.lib import osutils
  20 from chromite.lib import parallel
  21
  22
  23 class CleanUpStage(generic_stages.BuilderStage):
  24   """Stages that cleans up build artifacts from previous runs.
  25
  26   This stage cleans up previous KVM state, temporary git commits,
  27   clobbers, and wipes tmp inside the chroot.
  28   """
  29
  30   option_name = 'clean'
  31
  32   def _CleanChroot(self):
  33     commands.CleanupChromeKeywordsFile(self._boards,
  34                                        self._build_root)
  35     chroot_tmpdir = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR,
  36                                  'tmp')
  37     if os.path.exists(chroot_tmpdir):
  38       cros_build_lib.SudoRunCommand(['rm', '-rf', chroot_tmpdir],
  39                                     print_cmd=False)
  40       cros_build_lib.SudoRunCommand(['mkdir', '--mode', '1777', chroot_tmpdir],
  41                                     print_cmd=False)
  42
  43   def _DeleteChroot(self):
  44     chroot = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
  45     if os.path.exists(chroot):
  46       # At this stage, it's not safe to run the cros_sdk inside the buildroot
  47       # itself because we haven't sync'd yet, and the version of the chromite
  48       # in there might be broken. Since we've already unmounted everything in
  49       # there, we can just remove it using rm -rf.
  50       osutils.RmDir(chroot, ignore_missing=True, sudo=True)
  51
  52   def _DeleteArchivedTrybotImages(self):
  53     """Clear all previous archive images to save space."""
  54     for trybot in (False, True):
  55       archive_root = self._run.GetArchive().GetLocalArchiveRoot(trybot=trybot)
  56       osutils.RmDir(archive_root, ignore_missing=True)
  57
  58   def _DeleteArchivedPerfResults(self):
  59     """Clear any previously stashed perf results from hw testing."""
  60     for result in glob.glob(os.path.join(
  61         self._run.options.log_dir,
  62         '*.%s' % test_stages.HWTestStage.PERF_RESULTS_EXTENSION)):
  63       os.remove(result)
  64
  65   def _DeleteChromeBuildOutput(self):
  66     chrome_src = os.path.join(self._run.options.chrome_root, 'src')
  67     for out_dir in glob.glob(os.path.join(chrome_src, 'out_*')):
  68       osutils.RmDir(out_dir)
  69
  70   def _DeleteAutotestSitePackages(self):
  71     """Clears any previously downloaded site-packages."""
  72     site_packages_dir = os.path.join(self._build_root, 'src', 'third_party',
  73                                      'autotest', 'files', 'site-packages')
  74     # Note that these shouldn't be recreated but might be around from stale
  75     # builders.
  76     osutils.RmDir(site_packages_dir, ignore_missing=True)
  77
  78   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  79   def PerformStage(self):
  80     if (not (self._run.options.buildbot or self._run.options.remote_trybot)
  81         and self._run.options.clobber):
  82       if not commands.ValidateClobber(self._build_root):
  83         cros_build_lib.Die("--clobber in local mode must be approved.")
  84
  85     # If we can't get a manifest out of it, then it's not usable and must be
  86     # clobbered.
  87     manifest = None
  88     if not self._run.options.clobber:
  89       try:
  90         manifest = git.ManifestCheckout.Cached(self._build_root, search=False)
  91       except (KeyboardInterrupt, MemoryError, SystemExit):
  92         raise
  93       except Exception as e:
  94         # Either there is no repo there, or the manifest isn't usable.  If the
  95         # directory exists, log the exception for debugging reasons.  Either
  96         # way, the checkout needs to be wiped since it's in an unknown
  97         # state.
  98         if os.path.exists(self._build_root):
  99           cros_build_lib.Warning("ManifestCheckout at %s is unusable: %s",
 100                                  self._build_root, e)
 101
 102     # Clean mount points first to be safe about deleting.
 103     commands.CleanUpMountPoints(self._build_root)
 104
 105     if manifest is None:
 106       self._DeleteChroot()
 107       repository.ClearBuildRoot(self._build_root,
 108                                 self._run.options.preserve_paths)
 109     else:
 110       tasks = [functools.partial(commands.BuildRootGitCleanup,
 111                                  self._build_root),
 112                functools.partial(commands.WipeOldOutput, self._build_root),
 113                self._DeleteArchivedTrybotImages,
 114                self._DeleteArchivedPerfResults,
 115                self._DeleteAutotestSitePackages]
 116       if self._run.options.chrome_root:
 117         tasks.append(self._DeleteChromeBuildOutput)
 118       if self._run.config.chroot_replace and self._run.options.build:
 119         tasks.append(self._DeleteChroot)
 120       else:
 121         tasks.append(self._CleanChroot)
 122       parallel.RunParallelSteps(tasks)
 123
 124
 125 class InitSDKStage(generic_stages.BuilderStage):
 126   """Stage that is responsible for initializing the SDK."""
 127
 128   option_name = 'build'
 129
 130   def __init__(self, builder_run, chroot_replace=False, **kwargs):
 131     """InitSDK constructor.
 132
 133     Args:
 134       builder_run: Builder run instance for this run.
 135       chroot_replace: If True, force the chroot to be replaced.
 136     """
 137     super(InitSDKStage, self).__init__(builder_run, **kwargs)
 138     self.force_chroot_replace = chroot_replace
 139
 140   def PerformStage(self):
 141     chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
 142     replace = self._run.config.chroot_replace or self.force_chroot_replace
 143     pre_ver = post_ver = None
 144     if os.path.isdir(self._build_root) and not replace:
 145       try:
 146         pre_ver = cros_build_lib.GetChrootVersion(chroot=chroot_path)
 147         commands.RunChrootUpgradeHooks(self._build_root)
 148       except failures_lib.BuildScriptFailure:
 149         cros_build_lib.PrintBuildbotStepText('Replacing broken chroot')
 150         cros_build_lib.PrintBuildbotStepWarnings()
 151         replace = True
 152
 153     if not os.path.isdir(chroot_path) or replace:
 154       use_sdk = (self._run.config.use_sdk and not self._run.options.nosdk)
 155       pre_ver = None
 156       commands.MakeChroot(
 157           buildroot=self._build_root,
 158           replace=replace,
 159           use_sdk=use_sdk,
 160           chrome_root=self._run.options.chrome_root,
 161           extra_env=self._portage_extra_env)
 162
 163     post_ver = cros_build_lib.GetChrootVersion(chroot=chroot_path)
 164     if pre_ver is not None and pre_ver != post_ver:
 165       cros_build_lib.PrintBuildbotStepText('%s->%s' % (pre_ver, post_ver))
 166     else:
 167       cros_build_lib.PrintBuildbotStepText(post_ver)
 168
 169     commands.SetSharedUserPassword(
 170         self._build_root,
 171         password=self._run.config.shared_user_password)
 172
 173
 174 class SetupBoardStage(generic_stages.BoardSpecificBuilderStage, InitSDKStage):
 175   """Stage that is responsible for building host pkgs and setting up a board."""
 176
 177   option_name = 'build'
 178
 179   def PerformStage(self):
 180     # Calculate whether we should use binary packages.
 181     usepkg = (self._run.config.usepkg_setup_board and
 182               not self._latest_toolchain)
 183
 184     # We need to run chroot updates on most builders because they uprev after
 185     # the InitSDK stage. For the SDK builder, we can skip updates because uprev
 186     # is run prior to InitSDK. This is not just an optimization: It helps
 187     # workaround http://crbug.com/225509
 188     chroot_upgrade = (
 189       self._run.config.build_type != constants.CHROOT_BUILDER_TYPE)
 190
 191     # Iterate through boards to setup.
 192     chroot_path = os.path.join(self._build_root, constants.DEFAULT_CHROOT_DIR)
 193
 194     # Only update the board if we need to do so.
 195     board_path = os.path.join(chroot_path, 'build', self._current_board)
 196     if not os.path.isdir(board_path) or chroot_upgrade:
 197       commands.SetupBoard(
 198           self._build_root, board=self._current_board, usepkg=usepkg,
 199           chrome_binhost_only=self._run.config.chrome_binhost_only,
 200           force=self._run.config.board_replace,
 201           extra_env=self._portage_extra_env, chroot_upgrade=chroot_upgrade,
 202           profile=self._run.options.profile or self._run.config.profile)
 203
 204
 205 class BuildPackagesStage(generic_stages.BoardSpecificBuilderStage,
 206                          generic_stages.ArchivingStageMixin):
 207   """Build Chromium OS packages."""
 208
 209   option_name = 'build'
 210   def __init__(self, builder_run, board, afdo_generate_min=False,
 211                afdo_use=False, **kwargs):
 212     super(BuildPackagesStage, self).__init__(builder_run, board, **kwargs)
 213     self._afdo_generate_min = afdo_generate_min
 214     assert not afdo_generate_min or not afdo_use
 215
 216     useflags = self._run.config.useflags[:]
 217     if afdo_use:
 218       self.name += ' [%s]' % constants.USE_AFDO_USE
 219       useflags.append(constants.USE_AFDO_USE)
 220
 221     if useflags:
 222       self._portage_extra_env.setdefault('USE', '')
 223       self._portage_extra_env['USE'] += ' ' + ' '.join(useflags)
 224
 225   def PerformStage(self):
 226     # If we have rietveld patches, always compile Chrome from source.
 227     noworkon = not self._run.options.rietveld_patches
 228
 229     commands.Build(self._build_root,
 230                    self._current_board,
 231                    build_autotest=self._run.ShouldBuildAutotest(),
 232                    usepkg=self._run.config.usepkg_build_packages,
 233                    chrome_binhost_only=self._run.config.chrome_binhost_only,
 234                    packages=self._run.config.packages,
 235                    skip_chroot_upgrade=True,
 236                    chrome_root=self._run.options.chrome_root,
 237                    noworkon=noworkon,
 238                    extra_env=self._portage_extra_env)
 239
 240
 241 class BuildImageStage(BuildPackagesStage):
 242   """Build standard Chromium OS images."""
 243
 244   option_name = 'build'
 245   config_name = 'images'
 246
 247   def _BuildImages(self):
 248     # We only build base, dev, and test images from this stage.
 249     if self._afdo_generate_min:
 250       images_can_build = set(['test'])
 251     else:
 252       images_can_build = set(['base', 'dev', 'test'])
 253     images_to_build = set(self._run.config.images).intersection(
 254         images_can_build)
 255
 256     version = self._run.attrs.release_tag
 257     disk_layout = self._run.config.disk_layout
 258     if self._afdo_generate_min and version:
 259       version = '%s-afdo-generate' % version
 260
 261     rootfs_verification = self._run.config.rootfs_verification
 262     commands.BuildImage(self._build_root,
 263                         self._current_board,
 264                         sorted(images_to_build),
 265                         rootfs_verification=rootfs_verification,
 266                         version=version,
 267                         disk_layout=disk_layout,
 268                         extra_env=self._portage_extra_env)
 269
 270     # Update link to latest image.
 271     latest_image = os.readlink(self.GetImageDirSymlink('latest'))
 272     cbuildbot_image_link = self.GetImageDirSymlink()
 273     if os.path.lexists(cbuildbot_image_link):
 274       os.remove(cbuildbot_image_link)
 275
 276     os.symlink(latest_image, cbuildbot_image_link)
 277
 278     self.board_runattrs.SetParallel('images_generated', True)
 279
 280     parallel.RunParallelSteps(
 281         [self._BuildVMImage, lambda: self._GenerateAuZip(cbuildbot_image_link)])
 282
 283   def _BuildVMImage(self):
 284     if self._run.config.vm_tests and not self._afdo_generate_min:
 285       commands.BuildVMImageForTesting(
 286           self._build_root,
 287           self._current_board,
 288           disk_layout=self._run.config.disk_vm_layout,
 289           extra_env=self._portage_extra_env)
 290
 291   def _GenerateAuZip(self, image_dir):
 292     """Create au-generator.zip."""
 293     if not self._afdo_generate_min:
 294       commands.GenerateAuZip(self._build_root,
 295                              image_dir,
 296                              extra_env=self._portage_extra_env)
 297
 298   def _HandleStageException(self, exc_info):
 299     """Tell other stages to not wait on us if we die for some reason."""
 300     self.board_runattrs.SetParallelDefault('images_generated', False)
 301     return super(BuildImageStage, self)._HandleStageException(exc_info)
 302
 303   def PerformStage(self):
 304     self._BuildImages()
 305
 306
 307 class UprevStage(generic_stages.BuilderStage):
 308   """Stage that uprevs Chromium OS packages that the builder intends to
 309   validate.
 310   """
 311
 312   config_name = 'uprev'
 313   option_name = 'uprev'
 314
 315   def __init__(self, builder_run, boards=None, enter_chroot=True, **kwargs):
 316     super(UprevStage, self).__init__(builder_run, **kwargs)
 317     self._enter_chroot = enter_chroot
 318     if boards is not None:
 319       self._boards = boards
 320
 321   def PerformStage(self):
 322     # Perform other uprevs.
 323     overlays, _ = self._ExtractOverlays()
 324     commands.UprevPackages(self._build_root,
 325                            self._boards,
 326                            overlays,
 327                            enter_chroot=self._enter_chroot)