src/third_party/chromite/cros/commands/cros_chrome_sdk_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """This module tests the cros image command."""
   7
   8 from __future__ import print_function
   9
  10 import copy
  11 import mock
  12 import os
  13 import shutil
  14 import sys
  15
  16 sys.path.insert(0, os.path.abspath('%s/../../..' % os.path.dirname(__file__)))
  17 from chromite.cbuildbot import constants
  18 from chromite.cros.commands import cros_chrome_sdk
  19 from chromite.cros.commands import init_unittest
  20 from chromite.lib import cache
  21 from chromite.lib import chrome_util
  22 from chromite.lib import cros_build_lib_unittest
  23 from chromite.lib import cros_test_lib
  24 from chromite.lib import gs
  25 from chromite.lib import gs_unittest
  26 from chromite.lib import osutils
  27 from chromite.lib import partial_mock
  28
  29 # pylint: disable=W0212
  30
  31 class MockChromeSDKCommand(init_unittest.MockCommand):
  32   """Mock out the build command."""
  33   TARGET = 'chromite.cros.commands.cros_chrome_sdk.ChromeSDKCommand'
  34   TARGET_CLASS = cros_chrome_sdk.ChromeSDKCommand
  35   COMMAND = 'chrome-sdk'
  36   ATTRS = ('_GOMA_URL', '_SetupEnvironment') + init_unittest.MockCommand.ATTRS
  37
  38   _GOMA_URL = 'Invalid URL'
  39
  40   def __init__(self, *args, **kwargs):
  41     init_unittest.MockCommand.__init__(self, *args, **kwargs)
  42     self.env = None
  43
  44   def _SetupEnvironment(self, *args, **kwargs):
  45     env = self.backup['_SetupEnvironment'](*args, **kwargs)
  46     self.env = copy.deepcopy(env)
  47     return env
  48
  49
  50 class ParserTest(cros_test_lib.MockTempDirTestCase):
  51   """Test the parser."""
  52   def testNormal(self):
  53     """Tests that our example parser works normally."""
  54     with MockChromeSDKCommand(
  55         ['--board', SDKFetcherMock.BOARD],
  56         base_args=['--cache-dir', self.tempdir]) as bootstrap:
  57       self.assertEquals(bootstrap.inst.options.board, SDKFetcherMock.BOARD)
  58       self.assertEquals(bootstrap.inst.options.cache_dir, self.tempdir)
  59
  60
  61 def _GSCopyMock(_self, path, dest, **_kwargs):
  62   """Used to simulate a GS Copy operation."""
  63   with osutils.TempDir() as tempdir:
  64     local_path = os.path.join(tempdir, os.path.basename(path))
  65     osutils.Touch(local_path)
  66     shutil.move(local_path, dest)
  67
  68
  69 def _DependencyMockCtx(f):
  70   """Attribute that ensures dependency PartialMocks are started.
  71
  72   Since PartialMock does not support nested mocking, we need to first call
  73   stop() on the outer level PartialMock (which is passed in to us).  We then
  74   re-start() the outer level upon exiting the context.
  75   """
  76   def new_f(self, *args, **kwargs):
  77     if not self.entered:
  78       try:
  79         self.entered = True
  80         # Temporarily disable outer GSContext mock before starting our mock.
  81         # TODO(rcui): Generalize this attribute and include in partial_mock.py.
  82         for emock in self.external_mocks:
  83           emock.stop()
  84
  85         with self.gs_mock:
  86           return f(self, *args, **kwargs)
  87       finally:
  88         self.entered = False
  89         for emock in self.external_mocks:
  90           emock.start()
  91     else:
  92       return f(self, *args, **kwargs)
  93   return new_f
  94
  95
  96 class SDKFetcherMock(partial_mock.PartialMock):
  97   """Provides mocking functionality for SDKFetcher."""
  98
  99   TARGET = 'chromite.cros.commands.cros_chrome_sdk.SDKFetcher'
 100   ATTRS = ('__init__', 'GetFullVersion', '_GetMetadata', '_UpdateTarball',
 101            'UpdateDefaultVersion')
 102
 103   FAKE_METADATA = """
 104 {
 105   "boards": ["x86-alex"],
 106   "cros-version": "25.3543.2",
 107   "metadata-version": "1",
 108   "bot-hostname": "build82-m2.golo.chromium.org",
 109   "bot-config": "x86-alex-release",
 110   "toolchain-tuple": ["i686-pc-linux-gnu"],
 111   "toolchain-url": "2013/01/%(target)s-2013.01.23.003823.tar.xz",
 112   "sdk-version": "2013.01.23.003823"
 113 }"""
 114
 115   BOARD = 'x86-alex'
 116   VERSION = 'XXXX.X.X'
 117
 118   def __init__(self, external_mocks=None):
 119     """Initializes the mock.
 120
 121     Args:
 122       external_mocks: A list of already started PartialMock/patcher instances.
 123         stop() will be called on each element every time execution enters one of
 124         our the mocked out methods, and start() called on it once execution
 125         leaves the mocked out method.
 126     """
 127     partial_mock.PartialMock.__init__(self)
 128     self.external_mocks = external_mocks or []
 129     self.entered = False
 130     self.gs_mock = gs_unittest.GSContextMock()
 131     self.gs_mock.SetDefaultCmdResult()
 132     self.env = None
 133
 134   @_DependencyMockCtx
 135   def _target__init__(self, inst, *args, **kwargs):
 136     self.backup['__init__'](inst, *args, **kwargs)
 137     if not inst.cache_base.startswith('/tmp'):
 138       raise AssertionError('For testing, SDKFetcher cache_dir needs to be a '
 139                            'dir under /tmp')
 140
 141   @_DependencyMockCtx
 142   def UpdateDefaultVersion(self, inst, *_args, **_kwargs):
 143     inst._SetDefaultVersion(self.VERSION)
 144     return self.VERSION, True
 145
 146   @_DependencyMockCtx
 147   def _UpdateTarball(self, inst, *args, **kwargs):
 148     with mock.patch.object(gs.GSContext, 'Copy', autospec=True,
 149                            side_effect=_GSCopyMock):
 150       with mock.patch.object(cache, 'Untar'):
 151         return self.backup['_UpdateTarball'](inst, *args, **kwargs)
 152
 153   @_DependencyMockCtx
 154   def GetFullVersion(self, _inst, version):
 155     return 'R26-%s' % version
 156
 157   @_DependencyMockCtx
 158   def _GetMetadata(self, inst, *args, **kwargs):
 159     self.gs_mock.SetDefaultCmdResult()
 160     self.gs_mock.AddCmdResult(
 161         partial_mock.ListRegex('cat .*/%s' % constants.METADATA_JSON),
 162         output=self.FAKE_METADATA)
 163     return self.backup['_GetMetadata'](inst, *args, **kwargs)
 164
 165
 166 class RunThroughTest(cros_test_lib.MockTempDirTestCase,
 167                      cros_test_lib.LoggingTestCase):
 168   """Run the script with most things mocked out."""
 169
 170   VERSION_KEY = (SDKFetcherMock.BOARD, SDKFetcherMock.VERSION,
 171                  constants.CHROME_SYSROOT_TAR)
 172
 173   FAKE_ENV = {
 174       'GYP_DEFINES': "sysroot='/path/to/sysroot'",
 175       'CXX': 'x86_64-cros-linux-gnu-g++ -B /path/to/gold',
 176       'CC': 'x86_64-cros-linux-gnu-gcc -B /path/to/gold',
 177       'LD': 'x86_64-cros-linux-gnu-g++ -B /path/to/gold',
 178   }
 179
 180   def SetupCommandMock(self, extra_args=None):
 181     cmd_args = ['--board', SDKFetcherMock.BOARD, '--chrome-src',
 182                 self.chrome_src_dir, 'true']
 183     if extra_args:
 184       cmd_args.extend(extra_args)
 185
 186     self.cmd_mock = MockChromeSDKCommand(
 187         cmd_args, base_args=['--cache-dir', self.tempdir])
 188     self.StartPatcher(self.cmd_mock)
 189     self.cmd_mock.UnMockAttr('Run')
 190
 191   def SourceEnvironmentMock(self, path, *_args, **_kwargs):
 192     if path.endswith('environment'):
 193       return copy.deepcopy(self.FAKE_ENV)
 194     return {}
 195
 196   def setUp(self):
 197     self.rc_mock = cros_build_lib_unittest.RunCommandMock()
 198     self.rc_mock.SetDefaultCmdResult()
 199     self.StartPatcher(self.rc_mock)
 200
 201     self.sdk_mock = self.StartPatcher(SDKFetcherMock(
 202         external_mocks=[self.rc_mock]))
 203
 204     # This needs to occur before initializing MockChromeSDKCommand.
 205     self.bashrc = os.path.join(self.tempdir, 'bashrc')
 206     self.PatchObject(constants, 'CHROME_SDK_BASHRC', new=self.bashrc)
 207
 208     self.PatchObject(osutils, 'SourceEnvironment',
 209                      autospec=True, side_effect=self.SourceEnvironmentMock)
 210     self.rc_mock.AddCmdResult(cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD,
 211                               output='8088')
 212
 213     # Initialized by SetupCommandMock.
 214     self.cmd_mock = None
 215
 216     # Set up a fake Chrome src/ directory
 217     self.chrome_root = os.path.join(self.tempdir, 'chrome_root')
 218     self.chrome_src_dir = os.path.join(self.chrome_root, 'src')
 219     osutils.SafeMakedirs(self.chrome_src_dir)
 220     osutils.Touch(os.path.join(self.chrome_root, '.gclient'))
 221
 222   @property
 223   def cache(self):
 224     return self.cmd_mock.inst.sdk.tarball_cache
 225
 226   def testIt(self):
 227     """Test a runthrough of the script."""
 228     self.SetupCommandMock()
 229     with cros_test_lib.LoggingCapturer() as logs:
 230       self.cmd_mock.inst.Run()
 231       self.AssertLogsContain(logs, 'Goma:', inverted=True)
 232
 233     with self.cache.Lookup(self.VERSION_KEY) as r:
 234       self.assertTrue(r.Exists())
 235
 236   def testErrorCodePassthrough(self):
 237     """Test that error codes are passed through."""
 238     self.SetupCommandMock()
 239     with cros_test_lib.LoggingCapturer():
 240       self.rc_mock.AddCmdResult(partial_mock.ListRegex('-- true'),
 241                                 returncode=5)
 242       returncode = self.cmd_mock.inst.Run()
 243       self.assertEquals(returncode, 5)
 244
 245   def testLocalSDKPath(self):
 246     """Fetch components from a local --sdk-path."""
 247     sdk_dir = os.path.join(self.tempdir, 'sdk_dir')
 248     osutils.SafeMakedirs(sdk_dir)
 249     osutils.WriteFile(os.path.join(sdk_dir, constants.METADATA_JSON),
 250                       SDKFetcherMock.FAKE_METADATA)
 251     self.SetupCommandMock(extra_args=['--sdk-path', sdk_dir])
 252     with cros_test_lib.LoggingCapturer():
 253       self.cmd_mock.inst.Run()
 254
 255   def testGomaError(self):
 256     """We print an error message when GomaError is raised."""
 257     self.SetupCommandMock()
 258     with cros_test_lib.LoggingCapturer() as logs:
 259       self.PatchObject(cros_chrome_sdk.ChromeSDKCommand, '_FetchGoma',
 260                        side_effect=cros_chrome_sdk.GomaError())
 261       self.cmd_mock.inst.Run()
 262       self.AssertLogsContain(logs, 'Goma:')
 263
 264   def testSpecificComponent(self):
 265     """Tests that SDKFetcher.Prepare() handles |components| param properly."""
 266     sdk = cros_chrome_sdk.SDKFetcher(os.path.join(self.tempdir),
 267                                     SDKFetcherMock.BOARD)
 268     components = [constants.BASE_IMAGE_TAR, constants.CHROME_SYSROOT_TAR]
 269     with sdk.Prepare(components=components) as ctx:
 270       for c in components:
 271         self.assertTrue(os.path.exists(ctx.key_map[c].path))
 272       for c in [constants.IMAGE_SCRIPTS_TAR, constants.CHROME_ENV_TAR]:
 273         self.assertFalse(c in ctx.key_map)
 274
 275   @staticmethod
 276   def FindInPath(paths, endswith):
 277     for path in paths.split(':'):
 278       if path.endswith(endswith):
 279         return True
 280     return False
 281
 282   def testGomaInPath(self, inverted=False):
 283     """Verify that we do indeed add Goma to the PATH."""
 284     extra_args = ['--nogoma'] if inverted else None
 285     self.SetupCommandMock(extra_args)
 286     self.cmd_mock.inst.Run()
 287
 288     assert_fn = self.assertNotIn if inverted else self.assertIn
 289     gyp_defines_str = self.cmd_mock.env['GYP_DEFINES']
 290     gyp_defines = chrome_util.ProcessGypDefines(gyp_defines_str)
 291     assert_fn('gomadir', gyp_defines)
 292     assert_fn('use_goma', gyp_defines)
 293
 294   def testNoGoma(self):
 295     """Verify that we do not add Goma to the PATH."""
 296     self.testGomaInPath(inverted=True)
 297
 298   def testClang(self):
 299     """Verifies clang codepath."""
 300     with cros_test_lib.LoggingCapturer():
 301       self.SetupCommandMock(extra_args=['--clang'])
 302       self.cmd_mock.inst.Run()
 303
 304
 305 class GomaTest(cros_test_lib.MockTempDirTestCase,
 306                cros_test_lib.LoggingTestCase):
 307   """Test Goma setup functionality."""
 308
 309   def setUp(self):
 310     self.rc_mock = cros_build_lib_unittest.RunCommandMock()
 311     self.rc_mock.SetDefaultCmdResult()
 312     self.StartPatcher(self.rc_mock)
 313
 314     self.cmd_mock = MockChromeSDKCommand(
 315         ['--board', SDKFetcherMock.BOARD, 'true'],
 316         base_args=['--cache-dir', self.tempdir])
 317     self.StartPatcher(self.cmd_mock)
 318
 319   def VerifyGomaError(self):
 320     self.assertRaises(cros_chrome_sdk.GomaError, self.cmd_mock.inst._FetchGoma)
 321
 322   def testNoGomaPort(self):
 323     """We print an error when gomacc is not returning a port."""
 324     self.rc_mock.AddCmdResult(
 325           cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD)
 326     self.VerifyGomaError()
 327
 328   def testGomaccError(self):
 329     """We print an error when gomacc exits with nonzero returncode."""
 330     self.rc_mock.AddCmdResult(
 331         cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD, returncode=1)
 332     self.VerifyGomaError()
 333
 334   def testFetchError(self):
 335     """We print an error when we can't fetch Goma."""
 336     self.rc_mock.AddCmdResult(
 337         cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD, returncode=1)
 338     self.VerifyGomaError()
 339
 340   def testGomaStart(self):
 341     """Test that we start Goma if it's not already started."""
 342     # Duplicate return values.
 343     self.PatchObject(cros_chrome_sdk.ChromeSDKCommand, '_GomaPort',
 344                      side_effect=['XXXX', 'XXXX'])
 345     # Run it twice to exercise caching.
 346     for _ in range(2):
 347       goma_dir, goma_port = self.cmd_mock.inst._FetchGoma()
 348       self.assertEquals(goma_port, 'XXXX')
 349       self.assertTrue(bool(goma_dir))
 350
 351
 352 class VersionTest(cros_test_lib.MockTempDirTestCase):
 353   """Tests the determination of which SDK version to use."""
 354
 355   VERSION = '3543.2.0'
 356   FULL_VERSION = 'R55-%s' % VERSION
 357   BOARD = 'lumpy'
 358
 359   VERSION_BASE = ('gs://chromeos-image-archive/%s-release/LATEST-%s'
 360                   % (BOARD, VERSION))
 361
 362   CAT_ERROR = 'CommandException: No URLs matched %s' % VERSION_BASE
 363   LS_ERROR = 'CommandException: One or more URLs matched no objects.'
 364
 365   def setUp(self):
 366     self.gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
 367     self.gs_mock.SetDefaultCmdResult()
 368     self.sdk_mock = self.StartPatcher(SDKFetcherMock(
 369         external_mocks=[self.gs_mock]))
 370
 371     os.environ.pop(cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV, None)
 372     self.sdk = cros_chrome_sdk.SDKFetcher(
 373         os.path.join(self.tempdir, 'cache'), self.BOARD)
 374
 375   def SetUpDefaultVersion(self, current, target, newest):
 376     self.PatchObject(cros_chrome_sdk.SDKFetcher, 'GetDefaultVersion',
 377                      return_value=current)
 378     self.PatchObject(cros_chrome_sdk.SDKFetcher, '_GetRepoCheckoutVersion',
 379                      return_value=target)
 380     self.PatchObject(cros_chrome_sdk.SDKFetcher, '_GetNewestManifestVersion',
 381                      return_value=newest)
 382     return self.sdk.UpdateDefaultVersion()
 383
 384   def testUpdateDefaultVersionNormal(self):
 385     """Updating default version with no cached default version."""
 386     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 387     target, updated = self.SetUpDefaultVersion(None, self.VERSION, '3544.0.0')
 388     self.assertEquals(target, self.VERSION)
 389     self.assertEquals(updated, True)
 390
 391   def testUpdateDefaultVersionTooNew(self):
 392     """Version in chromeos_version.sh isn't uploaded yet."""
 393     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 394     target, updated = self.SetUpDefaultVersion(None, '3543.10.0', self.VERSION)
 395     self.assertEquals(target, self.VERSION)
 396     self.assertEquals(updated, True)
 397
 398   def testUpdateDefaultVersionNoUpdate(self):
 399     """Nothing to update because the target version did not change."""
 400     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 401     target, updated = self.SetUpDefaultVersion(self.VERSION, self.VERSION,
 402                                                None)
 403     self.assertEquals(target, self.VERSION)
 404     self.assertEquals(updated, False)
 405
 406   def testUpdateDefaultChromeVersion(self):
 407     """We pick up the right LKGM version from the Chrome tree."""
 408     dir_struct = [
 409         'gclient_root/.gclient'
 410     ]
 411     cros_test_lib.CreateOnDiskHierarchy(self.tempdir, dir_struct)
 412     gclient_root = os.path.join(self.tempdir, 'gclient_root')
 413     self.PatchObject(os, 'getcwd', return_value=gclient_root)
 414
 415     lkgm_file = os.path.join(gclient_root, 'src', constants.PATH_TO_CHROME_LKGM)
 416     osutils.Touch(lkgm_file, makedirs=True)
 417     osutils.WriteFile(lkgm_file, self.VERSION)
 418     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 419     self.sdk.UpdateDefaultVersion()
 420     self.assertEquals(self.sdk.GetDefaultVersion(),
 421                       self.VERSION)
 422
 423   def testFullVersion(self):
 424     """Test full version calculation."""
 425     def RaiseException(*_args, **_kwargs):
 426       raise Exception('boom')
 427
 428     self.sdk_mock.UnMockAttr('GetFullVersion')
 429     self.gs_mock.AddCmdResult(
 430         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 431         output=self.FULL_VERSION)
 432     self.assertEquals(
 433         self.FULL_VERSION,
 434         self.sdk.GetFullVersion(self.VERSION))
 435     # Test that we access the cache on the next call, rather than checking GS.
 436     self.gs_mock.AddCmdResult(
 437         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 438         side_effect=RaiseException)
 439     self.assertEquals(
 440         self.FULL_VERSION,
 441         self.sdk.GetFullVersion(self.VERSION))
 442
 443   def testBadVersion(self):
 444     """We raise an exception for a bad version."""
 445     self.sdk_mock.UnMockAttr('GetFullVersion')
 446     self.gs_mock.AddCmdResult(
 447         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 448         output='', error=self.CAT_ERROR, returncode=1)
 449     self.gs_mock.AddCmdResult(
 450         partial_mock.ListRegex('ls .*%s' % self.VERSION),
 451         output='', error=self.LS_ERROR, returncode=1)
 452     self.assertRaises(cros_chrome_sdk.MissingSDK, self.sdk.GetFullVersion,
 453                       self.VERSION)
 454
 455   def testDefaultEnvBadBoard(self):
 456     """We don't use the version in the environment if board doesn't match."""
 457     os.environ[cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV] = self.VERSION
 458     self.assertNotEquals(self.VERSION, self.sdk_mock.VERSION)
 459     self.assertEquals(self.sdk.GetDefaultVersion(), None)
 460
 461   def testDefaultEnvGoodBoard(self):
 462     """We use the version in the environment if board matches."""
 463     sdk_version_env = cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV
 464     os.environ[sdk_version_env] = self.VERSION
 465     os.environ[cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV] = self.BOARD
 466     self.assertEquals(self.sdk.GetDefaultVersion(), self.VERSION)
 467
 468
 469 class PathVerifyTest(cros_test_lib.MockTempDirTestCase,
 470                      cros_test_lib.LoggingTestCase):
 471   """Tests user_rc PATH validation and warnings."""
 472
 473   def testPathVerifyWarnings(self):
 474     """Test the user rc PATH verification codepath."""
 475     def SourceEnvironmentMock(*_args, **_kwargs):
 476       return {
 477           'PATH': ':'.join([os.path.dirname(p) for p in abs_paths]),
 478       }
 479
 480     self.PatchObject(osutils, 'SourceEnvironment',
 481                      side_effect=SourceEnvironmentMock)
 482     file_list = (
 483       'goma/goma_ctl.py',
 484       'clang/clang',
 485       'chromite/parallel_emerge',
 486     )
 487     abs_paths = [os.path.join(self.tempdir, relpath) for relpath in file_list]
 488     for p in abs_paths:
 489       osutils.Touch(p, makedirs=True, mode=0o755)
 490
 491     with cros_test_lib.LoggingCapturer() as logs:
 492       cros_chrome_sdk.ChromeSDKCommand._VerifyGoma(None)
 493       cros_chrome_sdk.ChromeSDKCommand._VerifyChromiteBin(None)
 494
 495     for msg in ['managed Goma', 'default Chromite']:
 496       self.AssertLogsMatch(logs, msg)
 497
 498
 499 if __name__ == '__main__':
 500   cros_test_lib.main()