src/third_party/chromite/cbuildbot/stages/test_stages.py

   1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Module containing the test stages."""
   6
   7 import collections
   8 import logging
   9 import os
  10
  11 from chromite.cbuildbot import commands
  12 from chromite.cbuildbot import cbuildbot_config
  13 from chromite.cbuildbot import failures_lib
  14 from chromite.cbuildbot import constants
  15 from chromite.cbuildbot import lab_status
  16 from chromite.cbuildbot import validation_pool
  17 from chromite.cbuildbot.stages import generic_stages
  18 from chromite.cros.tests import image_test
  19 from chromite.lib import cgroups
  20 from chromite.lib import cros_build_lib
  21 from chromite.lib import osutils
  22 from chromite.lib import perf_uploader
  23 from chromite.lib import retry_util
  24 from chromite.lib import timeout_util
  25
  26
  27 _VM_TEST_ERROR_MSG = """
  28 !!!VMTests failed!!!
  29
  30 Logs are uploaded in the corresponding %(vm_test_results)s. This can be found
  31 by clicking on the artifacts link in the "Report" Stage. Specifically look
  32 for the test_harness/failed for the failing tests. For more
  33 particulars, please refer to which test failed i.e. above see the
  34 individual test that failed -- or if an update failed, check the
  35 corresponding update directory.
  36 """
  37 PRE_CQ = validation_pool.PRE_CQ
  38
  39 CQ_HWTEST_WAS_ABORTED = ('HWTest was aborted, because another commit '
  40                          'queue builder failed outside of HWTest.')
  41
  42
  43 class UnitTestStage(generic_stages.BoardSpecificBuilderStage):
  44   """Run unit tests."""
  45
  46   option_name = 'tests'
  47   config_name = 'unittests'
  48
  49   # If the unit tests take longer than 70 minutes, abort. They usually take
  50   # ten minutes to run.
  51   #
  52   # If the processes hang, parallel_emerge will print a status report after 60
  53   # minutes, so we picked 70 minutes because it gives us a little buffer time.
  54   UNIT_TEST_TIMEOUT = 70 * 60
  55
  56   def PerformStage(self):
  57     extra_env = {}
  58     if self._run.config.useflags:
  59       extra_env['USE'] = ' '.join(self._run.config.useflags)
  60     with timeout_util.Timeout(self.UNIT_TEST_TIMEOUT):
  61       commands.RunUnitTests(self._build_root,
  62                             self._current_board,
  63                             full=(not self._run.config.quick_unit),
  64                             blacklist=self._run.config.unittest_blacklist,
  65                             extra_env=extra_env)
  66
  67     if os.path.exists(os.path.join(self.GetImageDirSymlink(),
  68                                    'au-generator.zip')):
  69       commands.TestAuZip(self._build_root,
  70                          self.GetImageDirSymlink())
  71
  72
  73 class VMTestStage(generic_stages.BoardSpecificBuilderStage,
  74                   generic_stages.ArchivingStageMixin):
  75   """Run autotests in a virtual machine."""
  76
  77   option_name = 'tests'
  78   config_name = 'vm_tests'
  79
  80   VM_TEST_TIMEOUT = 60 * 60
  81
  82   def _PrintFailedTests(self, results_path, test_basename):
  83     """Print links to failed tests.
  84
  85     Args:
  86       results_path: Path to directory containing the test results.
  87       test_basename: The basename that the tests are archived to.
  88     """
  89     test_list = commands.ListFailedTests(results_path)
  90     for test_name, path in test_list:
  91       self.PrintDownloadLink(
  92           os.path.join(test_basename, path), text_to_display=test_name)
  93
  94   def _NoTestResults(self, path):
  95     """Returns True if |path| is not a directory or is an empty directory."""
  96     return not os.path.isdir(path) or not os.listdir(path)
  97
  98   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
  99   def _ArchiveTestResults(self, test_results_dir, test_basename):
 100     """Archives test results to Google Storage.
 101
 102     Args:
 103       test_results_dir: Name of the directory containing the test results.
 104       test_basename: The basename to archive the tests.
 105     """
 106     results_path = commands.GetTestResultsDir(
 107         self._build_root, test_results_dir)
 108
 109     # Skip archiving if results_path does not exist or is an empty directory.
 110     if self._NoTestResults(results_path):
 111       return
 112
 113     archived_results_dir = os.path.join(self.archive_path, test_basename)
 114     # Copy relevant files to archvied_results_dir.
 115     commands.ArchiveTestResults(results_path, archived_results_dir)
 116     upload_paths = [os.path.basename(archived_results_dir)]
 117     # Create the compressed tarball to upload.
 118     # TODO: We should revisit whether uploading the tarball is necessary.
 119     test_tarball = commands.BuildAndArchiveTestResultsTarball(
 120         archived_results_dir, self._build_root)
 121     upload_paths.append(test_tarball)
 122
 123     got_symbols = self.GetParallel('breakpad_symbols_generated',
 124                                    pretty_name='breakpad symbols')
 125     upload_paths += commands.GenerateStackTraces(
 126         self._build_root, self._current_board, test_results_dir,
 127         self.archive_path, got_symbols)
 128
 129     self._Upload(upload_paths)
 130     self._PrintFailedTests(results_path, test_basename)
 131
 132     # Remove the test results directory.
 133     osutils.RmDir(results_path, ignore_missing=True, sudo=True)
 134
 135   @failures_lib.SetFailureType(failures_lib.InfrastructureFailure)
 136   def _ArchiveVMFiles(self, test_results_dir):
 137     vm_files = commands.ArchiveVMFiles(
 138         self._build_root, os.path.join(test_results_dir, 'test_harness'),
 139         self.archive_path)
 140     # We use paths relative to |self.archive_path|, for prettier
 141     # formatting on the web page.
 142     self._Upload([os.path.basename(image) for image in vm_files])
 143
 144   def _Upload(self, filenames):
 145     cros_build_lib.Info('Uploading artifacts to Google Storage...')
 146     with self.ArtifactUploader(archive=False, strict=False) as queue:
 147       for filename in filenames:
 148         queue.put([filename])
 149         if filename.endswith('.dmp.txt'):
 150           prefix = 'crash: '
 151         elif constants.VM_DISK_PREFIX in os.path.basename(filename):
 152           prefix = 'vm_disk: '
 153         elif constants.VM_MEM_PREFIX in os.path.basename(filename):
 154           prefix = 'vm_memory: '
 155         else:
 156           prefix = ''
 157         self.PrintDownloadLink(filename, prefix)
 158
 159   def _RunTest(self, test_type, test_results_dir):
 160     """Run a VM test.
 161
 162     Args:
 163       test_type: Any test in constants.VALID_VM_TEST_TYPES
 164       test_results_dir: The base directory to store the results.
 165     """
 166     if test_type == constants.CROS_VM_TEST_TYPE:
 167       commands.RunCrosVMTest(self._current_board, self.GetImageDirSymlink())
 168     elif test_type == constants.DEV_MODE_TEST_TYPE:
 169       commands.RunDevModeTest(
 170         self._build_root, self._current_board, self.GetImageDirSymlink())
 171     else:
 172       commands.RunTestSuite(self._build_root,
 173                             self._current_board,
 174                             self.GetImageDirSymlink(),
 175                             os.path.join(test_results_dir,
 176                                          'test_harness'),
 177                             test_type=test_type,
 178                             whitelist_chrome_crashes=self._chrome_rev is None,
 179                             archive_dir=self.bot_archive_root)
 180
 181   def PerformStage(self):
 182     # These directories are used later to archive test artifacts.
 183     test_results_dir = commands.CreateTestRoot(self._build_root)
 184     test_basename = constants.VM_TEST_RESULTS % dict(attempt=self._attempt)
 185     try:
 186       for test_type in self._run.config.vm_tests:
 187         cros_build_lib.Info('Running VM test %s.', test_type)
 188         with cgroups.SimpleContainChildren('VMTest'):
 189           with timeout_util.Timeout(self.VM_TEST_TIMEOUT):
 190             self._RunTest(test_type, test_results_dir)
 191
 192     except Exception:
 193       cros_build_lib.Error(_VM_TEST_ERROR_MSG %
 194                            dict(vm_test_results=test_basename))
 195       self._ArchiveVMFiles(test_results_dir)
 196       raise
 197     finally:
 198       self._ArchiveTestResults(test_results_dir, test_basename)
 199
 200
 201 class HWTestStage(generic_stages.BoardSpecificBuilderStage,
 202                   generic_stages.ArchivingStageMixin):
 203   """Stage that runs tests in the Autotest lab."""
 204
 205   option_name = 'tests'
 206   config_name = 'hw_tests'
 207
 208   PERF_RESULTS_EXTENSION = 'results'
 209
 210   def __init__(self, builder_run, board, suite_config, **kwargs):
 211     super(HWTestStage, self).__init__(builder_run, board,
 212                                       suffix=' [%s]' % suite_config.suite,
 213                                       **kwargs)
 214     if not self._run.IsToTBuild():
 215       suite_config.SetBranchedValues()
 216
 217     self.suite_config = suite_config
 218     self.wait_for_results = True
 219
 220   @failures_lib.SetFailureType(failures_lib.GSFailure)
 221   def _CheckAborted(self):
 222     """Checks with GS to see if HWTest for this build's release_tag was aborted.
 223
 224     We currently only support aborting HWTests for the CQ, so this method only
 225     returns True for paladin builders.
 226
 227     Returns:
 228       True if HWTest have been aborted for this build's release_tag.
 229       False otherwise.
 230     """
 231     aborted = (cbuildbot_config.IsCQType(self._run.config.build_type) and
 232                commands.HaveCQHWTestsBeenAborted(self._run.GetVersion()))
 233     return aborted
 234
 235   # Disable complaint about calling _HandleStageException.
 236   # pylint: disable=W0212
 237   def _HandleStageException(self, exc_info):
 238     """Override and don't set status to FAIL but FORGIVEN instead."""
 239     exc_type = exc_info[0]
 240
 241     # If the suite config says HW Tests can only warn, only warn.
 242     if self.suite_config.warn_only:
 243       return self._HandleExceptionAsWarning(exc_info)
 244
 245     if self.suite_config.critical:
 246       return super(HWTestStage, self)._HandleStageException(exc_info)
 247
 248     aborted = False
 249     try:
 250       # _CheckAborted accesses Google Storage and could fail for many
 251       # reasons. Ignore any failures because we are already handling
 252       # exceptions.
 253       aborted = self._CheckAborted()
 254     except Exception:
 255       logging.warning('Unable to check whether HWTest was aborted.')
 256
 257     if aborted:
 258       # HWTest was aborted. This is only applicable to CQ.
 259       logging.warning(CQ_HWTEST_WAS_ABORTED)
 260       return self._HandleExceptionAsWarning(exc_info)
 261
 262     if issubclass(exc_type, commands.TestWarning):
 263       # HWTest passed with warning. All builders should pass.
 264       logging.warning('HWTest passed with warning code.')
 265       return self._HandleExceptionAsWarning(exc_info)
 266     elif (issubclass(exc_type, failures_lib.InfrastructureFailure) or
 267           issubclass(exc_type, commands.SuiteTimedOut)):
 268       # Tests did not run correctly or suite timed out before completion;
 269       # builders that do not check in code should pass. Note that timeout
 270       # could be caused by real bugs, but we ignore that for now so canaries
 271       # don't fail frequently due to timeouts.
 272       logging.warning('HWTest did not complete due to infrastructure issues '
 273                       '(%s)', exc_type)
 274       if not cbuildbot_config.IsPFQType(self._run.config.build_type):
 275         return self._HandleExceptionAsWarning(exc_info)
 276
 277     return super(HWTestStage, self)._HandleStageException(exc_info)
 278
 279   @failures_lib.SetFailureType(failures_lib.TestLabFailure)
 280   def _CheckLabStatus(self):
 281     """Checks whether lab is down or the boards has been disabled.
 282
 283     If tests cannot be run, raise an exception based on the reason.
 284     """
 285     lab_status.CheckLabStatus(self._current_board)
 286
 287   def PerformStage(self):
 288     if self._CheckAborted():
 289       cros_build_lib.PrintBuildbotStepText('aborted')
 290       cros_build_lib.Warning(CQ_HWTEST_WAS_ABORTED)
 291       return
 292
 293     build = '/'.join([self._bot_id, self.version])
 294     if self._run.options.remote_trybot and self._run.options.hwtest:
 295       debug = self._run.options.debug_forced
 296     else:
 297       debug = self._run.options.debug
 298
 299     self._CheckLabStatus()
 300     commands.RunHWTestSuite(build,
 301                             self.suite_config.suite,
 302                             self._current_board,
 303                             self.suite_config.pool,
 304                             self.suite_config.num,
 305                             self.suite_config.file_bugs,
 306                             self.wait_for_results,
 307                             self.suite_config.priority,
 308                             self.suite_config.timeout_mins,
 309                             self.suite_config.retry,
 310                             self.suite_config.minimum_duts,
 311                             debug)
 312
 313
 314 class AUTestStage(HWTestStage):
 315   """Stage for au hw test suites that requires special pre-processing."""
 316
 317   def PerformStage(self):
 318     """Wait for payloads to be staged and uploads its au control files."""
 319     with osutils.TempDir() as tempdir:
 320       tarball = commands.BuildAUTestTarball(
 321           self._build_root, self._current_board, tempdir,
 322           self.version, self.upload_url)
 323       self.UploadArtifact(tarball)
 324
 325     super(AUTestStage, self).PerformStage()
 326
 327
 328 class ASyncHWTestStage(HWTestStage, generic_stages.ForgivingBuilderStage):
 329   """Stage that fires and forgets hw test suites to the Autotest lab."""
 330
 331   def __init__(self, *args, **kwargs):
 332     super(ASyncHWTestStage, self).__init__(*args, **kwargs)
 333     self.wait_for_results = False
 334
 335
 336 class ImageTestStage(generic_stages.BoardSpecificBuilderStage,
 337                      generic_stages.ForgivingBuilderStage,
 338                      generic_stages.ArchivingStageMixin):
 339   """Stage that launches tests on the produced disk image."""
 340
 341   option_name = 'image_test'
 342   config_name = 'image_test'
 343
 344   # Give the tests 60 minutes to run. Image tests should be really quick but
 345   # the umount/rmdir bug (see osutils.UmountDir) may take a long time.
 346   IMAGE_TEST_TIMEOUT = 60 * 60
 347
 348   def __init__(self, *args, **kwargs):
 349     super(ImageTestStage, self).__init__(*args, **kwargs)
 350
 351   def PerformStage(self):
 352     test_results_dir = commands.CreateTestRoot(self._build_root)
 353     test_results_dir = os.path.join(test_results_dir, 'image_test_results')
 354     osutils.SafeMakedirs(test_results_dir)
 355     with timeout_util.Timeout(self.IMAGE_TEST_TIMEOUT):
 356       commands.RunTestImage(
 357           self._build_root,
 358           self._current_board,
 359           self.GetImageDirSymlink(),
 360           test_results_dir,
 361       )
 362     self.SendPerfValues(test_results_dir)
 363
 364   def SendPerfValues(self, test_results_dir):
 365     """Gather all perf values in |test_results_dir| and send them to chromeperf.
 366
 367     The uploading will be retried 3 times for each file.
 368
 369     Args:
 370       test_results_dir: A path to the directory with perf files.
 371     """
 372     # A dict of list of perf values, keyed by test name.
 373     perf_entries = collections.defaultdict(list)
 374     for root, _, filenames in os.walk(test_results_dir):
 375       for relative_name in filenames:
 376         if not image_test.IsPerfFile(relative_name):
 377           continue
 378         full_name = os.path.join(root, relative_name)
 379         entries = perf_uploader.LoadPerfValues(full_name)
 380         test_name = image_test.ImageTestCase.GetTestName(relative_name)
 381         perf_entries[test_name].extend(entries)
 382
 383     platform_name = self._run.bot_id
 384     cros_ver = self._run.GetVersionInfo(self._run.buildroot).VersionString()
 385     chrome_ver = self._run.DetermineChromeVersion()
 386     for test_name, perf_values in perf_entries.iteritems():
 387       retry_util.RetryException(perf_uploader.PerfUploadingError, 3,
 388                                 perf_uploader.UploadPerfValues,
 389                                 perf_values, platform_name, cros_ver,
 390                                 chrome_ver, test_name)