src/third_party/chromite/cbuildbot/stages/test_stages.py

   1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Module containing the test stages."""
   6
   7 from __future__ import print_function
   8
   9 import collections
  10 import logging
  11 import os
  12
  13 from chromite.cbuildbot import commands
  14 from chromite.cbuildbot import cbuildbot_config
  15 from chromite.cbuildbot import failures_lib
  16 from chromite.cbuildbot import constants
  17 from chromite.cbuildbot import lab_status
  18 from chromite.cbuildbot import validation_pool
  19 from chromite.cbuildbot.stages import generic_stages
  20 from chromite.lib import cgroups
  21 from chromite.lib import cros_build_lib
  22 from chromite.lib import osutils
  23 from chromite.lib import perf_uploader
  24 from chromite.lib import retry_util
  25 from chromite.lib import timeout_util
  26
  27
  28 _VM_TEST_ERROR_MSG = """
  29 !!!VMTests failed!!!
  30
  31 Logs are uploaded in the corresponding %(vm_test_results)s. This can be found
  32 by clicking on the artifacts link in the "Report" Stage. Specifically look
  33 for the test_harness/failed for the failing tests. For more
  34 particulars, please refer to which test failed i.e. above see the
  35 individual test that failed -- or if an update failed, check the
  36 corresponding update directory.
  37 """
  38 PRE_CQ = validation_pool.PRE_CQ
  39
  40 CQ_HWTEST_WAS_ABORTED = ('HWTest was aborted, because another commit '
  41                          'queue builder failed outside of HWTest.')
  42
  43
  44 class UnitTestStage(generic_stages.BoardSpecificBuilderStage):
  45   """Run unit tests."""
  46
  47   option_name = 'tests'
  48   config_name = 'unittests'
  49
  50   # If the unit tests take longer than 70 minutes, abort. They usually take
  51   # ten minutes to run.
  52   #
  53   # If the processes hang, parallel_emerge will print a status report after 60
  54   # minutes, so we picked 70 minutes because it gives us a little buffer time.
  55   UNIT_TEST_TIMEOUT = 70 * 60
  56
  57   def PerformStage(self):
  58     extra_env = {}
  59     if self._run.config.useflags:
  60       extra_env['USE'] = ' '.join(self._run.config.useflags)
  61     with timeout_util.Timeout(self.UNIT_TEST_TIMEOUT):
  62       commands.RunUnitTests(self._build_root,
  63                             self._current_board,
  64                             full=(not self._run.config.quick_unit),
  65                             blacklist=self._run.config.unittest_blacklist,
  66                             extra_env=extra_env)
  67
  68     if os.path.exists(os.path.join(self.GetImageDirSymlink(),
  69                                    'au-generator.zip')):
  70       commands.TestAuZip(self._build_root,
  71                          self.GetImageDirSymlink())
  72
  73
  74 class VMTestStage(generic_stages.BoardSpecificBuilderStage,
  75                   generic_stages.ArchivingStageMixin):
  76   """Run autotests in a virtual machine."""
  77
  78   option_name = 'tests'
  79   config_name = 'vm_tests'
  80
  81   VM_TEST_TIMEOUT = 60 * 60
  82
  83   def _PrintFailedTests(self, results_path, test_basename):
  84     """Print links to failed tests.
  85
  86     Args:
  87       results_path: Path to directory containing the test results.
  88       test_basename: The basename that the tests are archived to.
  89     """
  90     test_list = commands.ListFailedTests(results_path)
  91     for test_name, path in test_list:
  92       self.PrintDownloadLink(
  93           os.path.join(test_basename, path), text_to_display=test_name)
  94
  95   def _NoTestResults(self, path):
  96     """Returns True if |path| is not a directory or is an empty directory."""
  97     return not os.path.isdir(path) or not os.listdir(path)
  98
  99   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
 100   def _ArchiveTestResults(self, test_results_dir, test_basename):
 101     """Archives test results to Google Storage.
 102
 103     Args:
 104       test_results_dir: Name of the directory containing the test results.
 105       test_basename: The basename to archive the tests.
 106     """
 107     results_path = commands.GetTestResultsDir(
 108         self._build_root, test_results_dir)
 109
 110     # Skip archiving if results_path does not exist or is an empty directory.
 111     if self._NoTestResults(results_path):
 112       return
 113
 114     archived_results_dir = os.path.join(self.archive_path, test_basename)
 115     # Copy relevant files to archvied_results_dir.
 116     commands.ArchiveTestResults(results_path, archived_results_dir)
 117     upload_paths = [os.path.basename(archived_results_dir)]
 118     # Create the compressed tarball to upload.
 119     # TODO: We should revisit whether uploading the tarball is necessary.
 120     test_tarball = commands.BuildAndArchiveTestResultsTarball(
 121         archived_results_dir, self._build_root)
 122     upload_paths.append(test_tarball)
 123
 124     got_symbols = self.GetParallel('breakpad_symbols_generated',
 125                                    pretty_name='breakpad symbols')
 126     upload_paths += commands.GenerateStackTraces(
 127         self._build_root, self._current_board, test_results_dir,
 128         self.archive_path, got_symbols)
 129
 130     self._Upload(upload_paths)
 131     self._PrintFailedTests(results_path, test_basename)
 132
 133     # Remove the test results directory.
 134     osutils.RmDir(results_path, ignore_missing=True, sudo=True)
 135
 136   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
 137   def _ArchiveVMFiles(self, test_results_dir):
 138     vm_files = commands.ArchiveVMFiles(
 139         self._build_root, os.path.join(test_results_dir, 'test_harness'),
 140         self.archive_path)
 141     # We use paths relative to |self.archive_path|, for prettier
 142     # formatting on the web page.
 143     self._Upload([os.path.basename(image) for image in vm_files])
 144
 145   def _Upload(self, filenames):
 146     cros_build_lib.Info('Uploading artifacts to Google Storage...')
 147     with self.ArtifactUploader(archive=False, strict=False) as queue:
 148       for filename in filenames:
 149         queue.put([filename])
 150         if filename.endswith('.dmp.txt'):
 151           prefix = 'crash: '
 152         elif constants.VM_DISK_PREFIX in os.path.basename(filename):
 153           prefix = 'vm_disk: '
 154         elif constants.VM_MEM_PREFIX in os.path.basename(filename):
 155           prefix = 'vm_memory: '
 156         else:
 157           prefix = ''
 158         self.PrintDownloadLink(filename, prefix)
 159
 160   def _RunTest(self, test_type, test_results_dir):
 161     """Run a VM test.
 162
 163     Args:
 164       test_type: Any test in constants.VALID_VM_TEST_TYPES
 165       test_results_dir: The base directory to store the results.
 166     """
 167     if test_type == constants.CROS_VM_TEST_TYPE:
 168       commands.RunCrosVMTest(self._current_board, self.GetImageDirSymlink())
 169     elif test_type == constants.DEV_MODE_TEST_TYPE:
 170       commands.RunDevModeTest(
 171         self._build_root, self._current_board, self.GetImageDirSymlink())
 172     else:
 173       commands.RunTestSuite(self._build_root,
 174                             self._current_board,
 175                             self.GetImageDirSymlink(),
 176                             os.path.join(test_results_dir,
 177                                          'test_harness'),
 178                             test_type=test_type,
 179                             whitelist_chrome_crashes=self._chrome_rev is None,
 180                             archive_dir=self.bot_archive_root)
 181
 182   def PerformStage(self):
 183     # These directories are used later to archive test artifacts.
 184     test_results_dir = commands.CreateTestRoot(self._build_root)
 185     test_basename = constants.VM_TEST_RESULTS % dict(attempt=self._attempt)
 186     try:
 187       for test_type in self._run.config.vm_tests:
 188         cros_build_lib.Info('Running VM test %s.', test_type)
 189         with cgroups.SimpleContainChildren('VMTest'):
 190           with timeout_util.Timeout(self.VM_TEST_TIMEOUT):
 191             self._RunTest(test_type, test_results_dir)
 192
 193     except Exception:
 194       cros_build_lib.Error(_VM_TEST_ERROR_MSG %
 195                            dict(vm_test_results=test_basename))
 196       self._ArchiveVMFiles(test_results_dir)
 197       raise
 198     finally:
 199       self._ArchiveTestResults(test_results_dir, test_basename)
 200
 201
 202 class HWTestStage(generic_stages.BoardSpecificBuilderStage,
 203                   generic_stages.ArchivingStageMixin):
 204   """Stage that runs tests in the Autotest lab."""
 205
 206   option_name = 'tests'
 207   config_name = 'hw_tests'
 208
 209   PERF_RESULTS_EXTENSION = 'results'
 210
 211   def __init__(self, builder_run, board, suite_config, **kwargs):
 212     super(HWTestStage, self).__init__(builder_run, board,
 213                                       suffix=' [%s]' % suite_config.suite,
 214                                       **kwargs)
 215     if not self._run.IsToTBuild():
 216       suite_config.SetBranchedValues()
 217
 218     self.suite_config = suite_config
 219     self.wait_for_results = True
 220
 221   @failures_lib.SetFailureType(failures_lib.GSFailure)
 222   def _CheckAborted(self):
 223     """Checks with GS to see if HWTest for this build's release_tag was aborted.
 224
 225     We currently only support aborting HWTests for the CQ, so this method only
 226     returns True for paladin builders.
 227
 228     Returns:
 229       True if HWTest have been aborted for this build's release_tag.
 230       False otherwise.
 231     """
 232     aborted = (cbuildbot_config.IsCQType(self._run.config.build_type) and
 233                commands.HaveCQHWTestsBeenAborted(self._run.GetVersion()))
 234     return aborted
 235
 236   # Disable complaint about calling _HandleStageException.
 237   # pylint: disable=W0212
 238   def _HandleStageException(self, exc_info):
 239     """Override and don't set status to FAIL but FORGIVEN instead."""
 240     exc_type = exc_info[0]
 241
 242     # If the suite config says HW Tests can only warn, only warn.
 243     if self.suite_config.warn_only:
 244       return self._HandleExceptionAsWarning(exc_info)
 245
 246     if self.suite_config.critical:
 247       return super(HWTestStage, self)._HandleStageException(exc_info)
 248
 249     aborted = False
 250     try:
 251       # _CheckAborted accesses Google Storage and could fail for many
 252       # reasons. Ignore any failures because we are already handling
 253       # exceptions.
 254       aborted = self._CheckAborted()
 255     except Exception:
 256       logging.warning('Unable to check whether HWTest was aborted.')
 257
 258     if aborted:
 259       # HWTest was aborted. This is only applicable to CQ.
 260       logging.warning(CQ_HWTEST_WAS_ABORTED)
 261       return self._HandleExceptionAsWarning(exc_info)
 262
 263     if issubclass(exc_type, commands.TestWarning):
 264       # HWTest passed with warning. All builders should pass.
 265       logging.warning('HWTest passed with warning code.')
 266       return self._HandleExceptionAsWarning(exc_info)
 267     elif issubclass(exc_type, commands.BoardNotAvailable):
 268       # Some boards may not have been setup in the lab yet for
 269       # non-code-checkin configs.
 270       if not cbuildbot_config.IsPFQType(self._run.config.build_type):
 271         logging.warning('HWTest did not run because the board was not '
 272                         'available in the lab yet')
 273         return self._HandleExceptionAsWarning(exc_info)
 274
 275     return super(HWTestStage, self)._HandleStageException(exc_info)
 276
 277   @failures_lib.SetFailureType(failures_lib.TestLabFailure)
 278   def _CheckLabStatus(self):
 279     """Checks whether lab is down or the boards has been disabled.
 280
 281     If tests cannot be run, raise an exception based on the reason.
 282     """
 283     lab_status.CheckLabStatus(self._current_board)
 284
 285   def PerformStage(self):
 286     if self._CheckAborted():
 287       cros_build_lib.PrintBuildbotStepText('aborted')
 288       cros_build_lib.Warning(CQ_HWTEST_WAS_ABORTED)
 289       return
 290
 291     build = '/'.join([self._bot_id, self.version])
 292     if self._run.options.remote_trybot and self._run.options.hwtest:
 293       debug = self._run.options.debug_forced
 294     else:
 295       debug = self._run.options.debug
 296
 297     self._CheckLabStatus()
 298     commands.RunHWTestSuite(build,
 299                             self.suite_config.suite,
 300                             self._current_board,
 301                             self.suite_config.pool,
 302                             self.suite_config.num,
 303                             self.suite_config.file_bugs,
 304                             self.wait_for_results,
 305                             self.suite_config.priority,
 306                             self.suite_config.timeout_mins,
 307                             self.suite_config.retry,
 308                             self.suite_config.minimum_duts,
 309                             debug)
 310
 311
 312 class AUTestStage(HWTestStage):
 313   """Stage for au hw test suites that requires special pre-processing."""
 314
 315   def PerformStage(self):
 316     """Wait for payloads to be staged and uploads its au control files."""
 317     with osutils.TempDir() as tempdir:
 318       tarball = commands.BuildAUTestTarball(
 319           self._build_root, self._current_board, tempdir,
 320           self.version, self.upload_url)
 321       self.UploadArtifact(tarball)
 322
 323     super(AUTestStage, self).PerformStage()
 324
 325
 326 class ASyncHWTestStage(HWTestStage, generic_stages.ForgivingBuilderStage):
 327   """Stage that fires and forgets hw test suites to the Autotest lab."""
 328
 329   def __init__(self, *args, **kwargs):
 330     super(ASyncHWTestStage, self).__init__(*args, **kwargs)
 331     self.wait_for_results = False
 332
 333
 334 class ImageTestStage(generic_stages.BoardSpecificBuilderStage,
 335                      generic_stages.ForgivingBuilderStage,
 336                      generic_stages.ArchivingStageMixin):
 337   """Stage that launches tests on the produced disk image."""
 338
 339   option_name = 'image_test'
 340   config_name = 'image_test'
 341
 342   # Give the tests 60 minutes to run. Image tests should be really quick but
 343   # the umount/rmdir bug (see osutils.UmountDir) may take a long time.
 344   IMAGE_TEST_TIMEOUT = 60 * 60
 345
 346   def __init__(self, *args, **kwargs):
 347     super(ImageTestStage, self).__init__(*args, **kwargs)
 348
 349   def PerformStage(self):
 350     test_results_dir = commands.CreateTestRoot(self._build_root)
 351     # CreateTestRoot returns a temp directory inside chroot.
 352     # We bring that back out to the build root.
 353     test_results_dir = os.path.join(self._build_root, test_results_dir[1:])
 354     test_results_dir = os.path.join(test_results_dir, 'image_test_results')
 355     osutils.SafeMakedirs(test_results_dir)
 356     try:
 357       with timeout_util.Timeout(self.IMAGE_TEST_TIMEOUT):
 358         commands.RunTestImage(
 359             self._build_root,
 360             self._current_board,
 361             self.GetImageDirSymlink(),
 362             test_results_dir,
 363         )
 364     finally:
 365       self.SendPerfValues(test_results_dir)
 366
 367   def SendPerfValues(self, test_results_dir):
 368     """Gather all perf values in |test_results_dir| and send them to chromeperf.
 369
 370     The uploading will be retried 3 times for each file.
 371
 372     Args:
 373       test_results_dir: A path to the directory with perf files.
 374     """
 375     # Import image_test here so that extra imports from image_test does not
 376     # affect cbuildbot in bootstrap.
 377     from chromite.cros.tests import image_test
 378     # A dict of list of perf values, keyed by test name.
 379     perf_entries = collections.defaultdict(list)
 380     for root, _, filenames in os.walk(test_results_dir):
 381       for relative_name in filenames:
 382         if not image_test.IsPerfFile(relative_name):
 383           continue
 384         full_name = os.path.join(root, relative_name)
 385         entries = perf_uploader.LoadPerfValues(full_name)
 386         test_name = image_test.ImageTestCase.GetTestName(relative_name)
 387         perf_entries[test_name].extend(entries)
 388
 389     platform_name = self._run.bot_id
 390     cros_ver = self._run.GetVersionInfo(self._run.buildroot).VersionString()
 391     chrome_ver = self._run.DetermineChromeVersion()
 392     for test_name, perf_values in perf_entries.iteritems():
 393       retry_util.RetryException(perf_uploader.PerfUploadingError, 3,
 394                                 perf_uploader.UploadPerfValues,
 395                                 perf_values, platform_name, cros_ver,
 396                                 chrome_ver, test_name)