src/third_party/chromite/cbuildbot/stages/chrome_stages.py

   1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Module containing the Chrome stages."""
   6
   7 from __future__ import print_function
   8
   9 import multiprocessing
  10 import platform
  11 import os
  12 import sys
  13
  14 from chromite.cbuildbot import commands
  15 from chromite.cbuildbot import failures_lib
  16 from chromite.cbuildbot import constants
  17 from chromite.cbuildbot import manifest_version
  18 from chromite.cbuildbot.stages import artifact_stages
  19 from chromite.cbuildbot.stages import generic_stages
  20 from chromite.cbuildbot.stages import sync_stages
  21 from chromite.lib import cros_build_lib
  22 from chromite.lib import git
  23 from chromite.lib import osutils
  24 from chromite.lib import parallel
  25
  26
  27 class SyncChromeStage(generic_stages.BuilderStage,
  28                       generic_stages.ArchivingStageMixin):
  29   """Stage that syncs Chrome sources if needed."""
  30
  31   option_name = 'managed_chrome'
  32
  33   def __init__(self, builder_run, **kwargs):
  34     super(SyncChromeStage, self).__init__(builder_run, **kwargs)
  35     # PerformStage() will fill this out for us.
  36     # TODO(mtennant): Replace with a run param.
  37     self.chrome_version = None
  38
  39   def HandleSkip(self):
  40     """Set run.attrs.chrome_version to chrome version in buildroot now."""
  41     self._run.attrs.chrome_version = self._run.DetermineChromeVersion()
  42     cros_build_lib.Debug('Existing chrome version is %s.',
  43                          self._run.attrs.chrome_version)
  44     self._WriteChromeVersionToMetadata()
  45     super(SyncChromeStage, self).HandleSkip()
  46
  47   def _GetChromeVersionFromMetadata(self):
  48     """Return the Chrome version from metadata; None if is does not exist."""
  49     version_dict = self._run.attrs.metadata.GetDict().get('version')
  50     return None if not version_dict else version_dict.get('chrome')
  51
  52   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  53   def PerformStage(self):
  54     if self._chrome_rev:
  55       if (self._chrome_rev == constants.CHROME_REV_SPEC and
  56           self._run.options.chrome_version):
  57         self.chrome_version = self._run.options.chrome_version
  58         cros_build_lib.Info(
  59             'Using chrome version from options.chrome_version: %s',
  60             self.chrome_version)
  61       else:
  62         self.chrome_version = self._GetChromeVersionFromMetadata()
  63         if self.chrome_version:
  64           cros_build_lib.Info(
  65               'Using chrome version from the metadata dictionary: %s',
  66               self.chrome_version)
  67
  68       # Perform chrome uprev.
  69       chrome_atom_to_build = None
  70       chrome_atom_to_build = commands.MarkChromeAsStable(
  71           self._build_root, self._run.manifest_branch,
  72           self._chrome_rev, self._boards,
  73           chrome_version=self.chrome_version)
  74
  75     kwargs = {}
  76     if self._chrome_rev == constants.CHROME_REV_SPEC:
  77       kwargs['revision'] = self.chrome_version
  78       cros_build_lib.PrintBuildbotStepText('revision %s' % kwargs['revision'])
  79     else:
  80       if not self.chrome_version:
  81         self.chrome_version = self._run.DetermineChromeVersion()
  82
  83       kwargs['tag'] = self.chrome_version
  84       cros_build_lib.PrintBuildbotStepText('tag %s' % kwargs['tag'])
  85
  86     useflags = self._run.config.useflags
  87     commands.SyncChrome(self._build_root, self._run.options.chrome_root,
  88                         useflags, **kwargs)
  89     if (self._chrome_rev and not chrome_atom_to_build and
  90         self._run.options.buildbot and
  91         self._run.config.build_type == constants.CHROME_PFQ_TYPE):
  92       cros_build_lib.Info('Chrome already uprevved')
  93       sys.exit(0)
  94
  95   def _WriteChromeVersionToMetadata(self):
  96     """Write chrome version to metadata and upload partial json file."""
  97     self._run.attrs.metadata.UpdateKeyDictWithDict(
  98         'version',
  99         {'chrome': self._run.attrs.chrome_version})
 100     self.UploadMetadata(filename=constants.PARTIAL_METADATA_JSON)
 101
 102   def _Finish(self):
 103     """Provide chrome_version to the rest of the run."""
 104     # Even if the stage failed, a None value for chrome_version still
 105     # means something.  In other words, this stage tried to run.
 106     self._run.attrs.chrome_version = self.chrome_version
 107     self._WriteChromeVersionToMetadata()
 108     super(SyncChromeStage, self)._Finish()
 109
 110
 111 class PatchChromeStage(generic_stages.BuilderStage):
 112   """Stage that applies Chrome patches if needed."""
 113
 114   option_name = 'rietveld_patches'
 115
 116   URL_BASE = 'https://codereview.chromium.org/%(id)s'
 117
 118   def PerformStage(self):
 119     for spatch in ' '.join(self._run.options.rietveld_patches).split():
 120       patch, colon, subdir = spatch.partition(':')
 121       if not colon:
 122         subdir = 'src'
 123       url = self.URL_BASE % {'id': patch}
 124       cros_build_lib.PrintBuildbotLink(spatch, url)
 125       commands.PatchChrome(self._run.options.chrome_root, patch, subdir)
 126
 127
 128 class ChromeSDKStage(generic_stages.BoardSpecificBuilderStage,
 129                      generic_stages.ArchivingStageMixin):
 130   """Run through the simple chrome workflow."""
 131
 132   def __init__(self, *args, **kwargs):
 133     super(ChromeSDKStage, self).__init__(*args, **kwargs)
 134     self._upload_queue = multiprocessing.Queue()
 135     self._pkg_dir = os.path.join(
 136         self._build_root, constants.DEFAULT_CHROOT_DIR,
 137         'build', self._current_board, 'var', 'db', 'pkg')
 138     if self._run.config.chrome_sdk_build_chrome:
 139       self.chrome_src = os.path.join(self._run.options.chrome_root, 'src')
 140       self.out_board_dir = os.path.join(
 141           self.chrome_src, 'out_%s' % self._current_board)
 142
 143   def _BuildAndArchiveChromeSysroot(self):
 144     """Generate and upload sysroot for building Chrome."""
 145     assert self.archive_path.startswith(self._build_root)
 146     extra_env = {}
 147     if self._run.config.useflags:
 148       extra_env['USE'] = ' '.join(self._run.config.useflags)
 149     in_chroot_path = git.ReinterpretPathForChroot(self.archive_path)
 150     cmd = ['cros_generate_sysroot', '--out-dir', in_chroot_path, '--board',
 151            self._current_board, '--package', constants.CHROME_CP]
 152     cros_build_lib.RunCommand(cmd, cwd=self._build_root, enter_chroot=True,
 153                               extra_env=extra_env)
 154     self._upload_queue.put([constants.CHROME_SYSROOT_TAR])
 155
 156   def _ArchiveChromeEbuildEnv(self):
 157     """Generate and upload Chrome ebuild environment."""
 158     chrome_dir = artifact_stages.ArchiveStage.SingleMatchGlob(
 159         os.path.join(self._pkg_dir, constants.CHROME_CP) + '-*')
 160     env_bzip = os.path.join(chrome_dir, 'environment.bz2')
 161     with osutils.TempDir(prefix='chrome-sdk-stage') as tempdir:
 162       # Convert from bzip2 to tar format.
 163       bzip2 = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)
 164       cros_build_lib.RunCommand(
 165           [bzip2, '-d', env_bzip, '-c'],
 166           log_stdout_to_file=os.path.join(tempdir, constants.CHROME_ENV_FILE))
 167       env_tar = os.path.join(self.archive_path, constants.CHROME_ENV_TAR)
 168       cros_build_lib.CreateTarball(env_tar, tempdir)
 169       self._upload_queue.put([os.path.basename(env_tar)])
 170
 171   def _VerifyChromeDeployed(self, tempdir):
 172     """Check to make sure deploy_chrome ran correctly."""
 173     if not os.path.exists(os.path.join(tempdir, 'chrome')):
 174       raise AssertionError('deploy_chrome did not run successfully!')
 175
 176   def _VerifySDKEnvironment(self):
 177     """Make sure the SDK environment is set up properly."""
 178     # If the environment wasn't set up, then the output directory wouldn't be
 179     # created after 'gclient runhooks'.
 180     # TODO: Make this check actually look at the environment.
 181     if not os.path.exists(self.out_board_dir):
 182       raise AssertionError('%s not created!' % self.out_board_dir)
 183
 184   def _BuildChrome(self, sdk_cmd):
 185     """Use the generated SDK to build Chrome."""
 186     # Validate fetching of the SDK and setting everything up.
 187     sdk_cmd.Run(['true'])
 188     # Actually build chromium.
 189     sdk_cmd.Run(['gclient', 'runhooks'])
 190     self._VerifySDKEnvironment()
 191     sdk_cmd.Ninja()
 192
 193   def _TestDeploy(self, sdk_cmd):
 194     """Test SDK deployment."""
 195     with osutils.TempDir(prefix='chrome-sdk-stage') as tempdir:
 196       # Use the TOT deploy_chrome.
 197       script_path = os.path.join(
 198           self._build_root, constants.CHROMITE_BIN_SUBDIR, 'deploy_chrome')
 199       sdk_cmd.Run([script_path, '--build-dir',
 200                    os.path.join(self.out_board_dir, 'Release'),
 201                    '--staging-only', '--staging-dir', tempdir])
 202       self._VerifyChromeDeployed(tempdir)
 203
 204   def _GenerateAndUploadMetadata(self):
 205     self.UploadMetadata(upload_queue=self._upload_queue,
 206                         filename=constants.PARTIAL_METADATA_JSON)
 207
 208   def PerformStage(self):
 209     if platform.dist()[-1] == 'lucid':
 210       # Chrome no longer builds on Lucid. See crbug.com/276311
 211       print('Ubuntu lucid is no longer supported.')
 212       print('Please upgrade to Ubuntu Precise.')
 213       cros_build_lib.PrintBuildbotStepWarnings()
 214       return
 215
 216     steps = [self._BuildAndArchiveChromeSysroot, self._ArchiveChromeEbuildEnv,
 217              self._GenerateAndUploadMetadata]
 218     with self.ArtifactUploader(self._upload_queue, archive=False):
 219       parallel.RunParallelSteps(steps)
 220
 221       if self._run.config.chrome_sdk_build_chrome:
 222         with osutils.TempDir(prefix='chrome-sdk-cache') as tempdir:
 223           cache_dir = os.path.join(tempdir, 'cache')
 224           extra_args = ['--cwd', self.chrome_src, '--sdk-path',
 225                         self.archive_path]
 226           sdk_cmd = commands.ChromeSDK(
 227               self._build_root, self._current_board, chrome_src=self.chrome_src,
 228               goma=self._run.config.chrome_sdk_goma,
 229               extra_args=extra_args, cache_dir=cache_dir)
 230           self._BuildChrome(sdk_cmd)
 231           self._TestDeploy(sdk_cmd)
 232
 233
 234 class ChromeLKGMSyncStage(sync_stages.SyncStage):
 235   """Stage that syncs to the last known good manifest for Chrome."""
 236
 237   output_manifest_sha1 = False
 238
 239   def GetNextManifest(self):
 240     """Override: Gets the LKGM from the Chrome tree."""
 241     chrome_lkgm = commands.GetChromeLKGM(self._run.options.chrome_version)
 242
 243     # We need a full buildspecs manager here as we need an initialized manifest
 244     # manager with paths to the spec.
 245     # TODO(mtennant): Consider registering as manifest_manager run param, for
 246     # consistency, but be careful that consumers do not get confused.
 247     # Currently only the "manifest_manager" from ManifestVersionedSync (and
 248     # subclasses) is used later in the flow.
 249     manifest_manager = manifest_version.BuildSpecsManager(
 250       source_repo=self.repo,
 251       manifest_repo=self._GetManifestVersionsRepoUrl(read_only=False),
 252       build_names=self._run.GetBuilderIds(),
 253       incr_type='build',
 254       force=False,
 255       branch=self._run.manifest_branch)
 256
 257     manifest_manager.BootstrapFromVersion(chrome_lkgm)
 258     return manifest_manager.GetLocalManifest(chrome_lkgm)