src/third_party/chromite/cros/commands/cros_chrome_sdk_unittest.py

   1 #!/usr/bin/python
   2
   3 # Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """This module tests the cros image command."""
   8
   9 import copy
  10 import mock
  11 import os
  12 import shutil
  13 import sys
  14
  15 sys.path.insert(0, os.path.abspath('%s/../../..' % os.path.dirname(__file__)))
  16 from chromite.buildbot import constants
  17 from chromite.cros.commands import cros_chrome_sdk
  18 from chromite.cros.commands import init_unittest
  19 from chromite.lib import cache
  20 from chromite.lib import chrome_util
  21 from chromite.lib import cros_build_lib_unittest
  22 from chromite.lib import cros_test_lib
  23 from chromite.lib import gs
  24 from chromite.lib import gs_unittest
  25 from chromite.lib import osutils
  26 from chromite.lib import partial_mock
  27
  28 # pylint: disable=W0212
  29
  30 class MockChromeSDKCommand(init_unittest.MockCommand):
  31   """Mock out the build command."""
  32   TARGET = 'chromite.cros.commands.cros_chrome_sdk.ChromeSDKCommand'
  33   TARGET_CLASS = cros_chrome_sdk.ChromeSDKCommand
  34   COMMAND = 'chrome-sdk'
  35   ATTRS = ('_GOMA_URL', '_SetupEnvironment') + init_unittest.MockCommand.ATTRS
  36
  37   _GOMA_URL = 'Invalid URL'
  38
  39   def __init__(self, *args, **kwargs):
  40     init_unittest.MockCommand.__init__(self, *args, **kwargs)
  41     self.env = None
  42
  43   def _SetupEnvironment(self, *args, **kwargs):
  44     env = self.backup['_SetupEnvironment'](*args, **kwargs)
  45     self.env = copy.deepcopy(env)
  46     return env
  47
  48
  49 class ParserTest(cros_test_lib.MockTempDirTestCase):
  50   """Test the parser."""
  51   def testNormal(self):
  52     """Tests that our example parser works normally."""
  53     with MockChromeSDKCommand(
  54         ['--board', SDKFetcherMock.BOARD],
  55         base_args=['--cache-dir', self.tempdir]) as bootstrap:
  56       self.assertEquals(bootstrap.inst.options.board, SDKFetcherMock.BOARD)
  57       self.assertEquals(bootstrap.inst.options.cache_dir, self.tempdir)
  58
  59
  60 def _GSCopyMock(_self, path, dest, **_kwargs):
  61   """Used to simulate a GS Copy operation."""
  62   with osutils.TempDir() as tempdir:
  63     local_path = os.path.join(tempdir, os.path.basename(path))
  64     osutils.Touch(local_path)
  65     shutil.move(local_path, dest)
  66
  67
  68 def _DependencyMockCtx(f):
  69   """Attribute that ensures dependency PartialMocks are started.
  70
  71   Since PartialMock does not support nested mocking, we need to first call
  72   stop() on the outer level PartialMock (which is passed in to us).  We then
  73   re-start() the outer level upon exiting the context.
  74   """
  75   def new_f(self, *args, **kwargs):
  76     if not self.entered:
  77       try:
  78         self.entered = True
  79         # Temporarily disable outer GSContext mock before starting our mock.
  80         # TODO(rcui): Generalize this attribute and include in partial_mock.py.
  81         for emock in self.external_mocks:
  82           emock.stop()
  83
  84         with self.gs_mock:
  85           return f(self, *args, **kwargs)
  86       finally:
  87         self.entered = False
  88         for emock in self.external_mocks:
  89           emock.start()
  90     else:
  91       return f(self, *args, **kwargs)
  92   return new_f
  93
  94
  95 class SDKFetcherMock(partial_mock.PartialMock):
  96   """Provides mocking functionality for SDKFetcher."""
  97
  98   TARGET = 'chromite.cros.commands.cros_chrome_sdk.SDKFetcher'
  99   ATTRS = ('__init__', 'GetFullVersion', '_GetMetadata', '_UpdateTarball',
 100            'UpdateDefaultVersion')
 101
 102   FAKE_METADATA = """
 103 {
 104   "boards": ["x86-alex"],
 105   "cros-version": "25.3543.2",
 106   "metadata-version": "1",
 107   "bot-hostname": "build82-m2.golo.chromium.org",
 108   "bot-config": "x86-alex-release",
 109   "toolchain-tuple": ["i686-pc-linux-gnu"],
 110   "toolchain-url": "2013/01/%(target)s-2013.01.23.003823.tar.xz",
 111   "sdk-version": "2013.01.23.003823"
 112 }"""
 113
 114   BOARD = 'x86-alex'
 115   VERSION = 'XXXX.X.X'
 116
 117   def __init__(self, external_mocks=None):
 118     """Initializes the mock.
 119
 120     Args:
 121       external_mocks: A list of already started PartialMock/patcher instances.
 122         stop() will be called on each element every time execution enters one of
 123         our the mocked out methods, and start() called on it once execution
 124         leaves the mocked out method.
 125     """
 126     partial_mock.PartialMock.__init__(self)
 127     self.external_mocks = external_mocks or []
 128     self.entered = False
 129     self.gs_mock = gs_unittest.GSContextMock()
 130     self.gs_mock.SetDefaultCmdResult()
 131     self.env = None
 132
 133   @_DependencyMockCtx
 134   def _target__init__(self, inst, *args, **kwargs):
 135     self.backup['__init__'](inst, *args, **kwargs)
 136     if not inst.cache_base.startswith('/tmp'):
 137       raise AssertionError('For testing, SDKFetcher cache_dir needs to be a '
 138                            'dir under /tmp')
 139
 140   @_DependencyMockCtx
 141   def UpdateDefaultVersion(self, inst, *_args, **_kwargs):
 142     inst._SetDefaultVersion(self.VERSION)
 143     return self.VERSION, True
 144
 145   @_DependencyMockCtx
 146   def _UpdateTarball(self, inst, *args, **kwargs):
 147     with mock.patch.object(gs.GSContext, 'Copy', autospec=True,
 148                            side_effect=_GSCopyMock):
 149       with mock.patch.object(cache, 'Untar'):
 150         return self.backup['_UpdateTarball'](inst, *args, **kwargs)
 151
 152   @_DependencyMockCtx
 153   def GetFullVersion(self, _inst, version):
 154     return 'R26-%s' % version
 155
 156   @_DependencyMockCtx
 157   def _GetMetadata(self, inst, *args, **kwargs):
 158     self.gs_mock.SetDefaultCmdResult()
 159     self.gs_mock.AddCmdResult(
 160         partial_mock.ListRegex('cat .*/%s' % constants.METADATA_JSON),
 161         output=self.FAKE_METADATA)
 162     return self.backup['_GetMetadata'](inst, *args, **kwargs)
 163
 164
 165 class RunThroughTest(cros_test_lib.MockTempDirTestCase,
 166                      cros_test_lib.LoggingTestCase):
 167   """Run the script with most things mocked out."""
 168
 169   VERSION_KEY = (SDKFetcherMock.BOARD, SDKFetcherMock.VERSION,
 170                  constants.CHROME_SYSROOT_TAR)
 171
 172   FAKE_ENV = {
 173       'GYP_DEFINES': "sysroot='/path/to/sysroot'",
 174       'CXX': 'x86_64-cros-linux-gnu-g++ -B /path/to/gold',
 175       'CC': 'x86_64-cros-linux-gnu-gcc -B /path/to/gold',
 176       'LD': 'x86_64-cros-linux-gnu-g++ -B /path/to/gold',
 177   }
 178
 179   def SetupCommandMock(self, extra_args=None):
 180     cmd_args = ['--board', SDKFetcherMock.BOARD, '--chrome-src',
 181                 self.chrome_src_dir, 'true']
 182     if extra_args:
 183       cmd_args.extend(extra_args)
 184
 185     self.cmd_mock = MockChromeSDKCommand(
 186         cmd_args, base_args=['--cache-dir', self.tempdir])
 187     self.StartPatcher(self.cmd_mock)
 188     self.cmd_mock.UnMockAttr('Run')
 189
 190   def SourceEnvironmentMock(self, path, *_args, **_kwargs):
 191     if path.endswith('environment'):
 192       return copy.deepcopy(self.FAKE_ENV)
 193     return {}
 194
 195   def setUp(self):
 196     self.rc_mock = cros_build_lib_unittest.RunCommandMock()
 197     self.rc_mock.SetDefaultCmdResult()
 198     self.StartPatcher(self.rc_mock)
 199
 200     self.sdk_mock = self.StartPatcher(SDKFetcherMock(
 201         external_mocks=[self.rc_mock]))
 202
 203     # This needs to occur before initializing MockChromeSDKCommand.
 204     self.bashrc = os.path.join(self.tempdir, 'bashrc')
 205     self.PatchObject(constants, 'CHROME_SDK_BASHRC', new=self.bashrc)
 206
 207     self.PatchObject(osutils, 'SourceEnvironment',
 208                      autospec=True, side_effect=self.SourceEnvironmentMock)
 209     self.rc_mock.AddCmdResult(cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD,
 210                               output='8088')
 211
 212     # Initialized by SetupCommandMock.
 213     self.cmd_mock = None
 214
 215     # Set up a fake Chrome src/ directory
 216     self.chrome_root = os.path.join(self.tempdir, 'chrome_root')
 217     self.chrome_src_dir = os.path.join(self.chrome_root, 'src')
 218     osutils.SafeMakedirs(self.chrome_src_dir)
 219     osutils.Touch(os.path.join(self.chrome_root, '.gclient'))
 220
 221   @property
 222   def cache(self):
 223     return self.cmd_mock.inst.sdk.tarball_cache
 224
 225   def testIt(self):
 226     """Test a runthrough of the script."""
 227     self.SetupCommandMock()
 228     with cros_test_lib.LoggingCapturer() as logs:
 229       self.cmd_mock.inst.Run()
 230       self.AssertLogsContain(logs, 'Goma:', inverted=True)
 231
 232     with self.cache.Lookup(self.VERSION_KEY) as r:
 233       self.assertTrue(r.Exists())
 234
 235   def testErrorCodePassthrough(self):
 236     """Test that error codes are passed through."""
 237     self.SetupCommandMock()
 238     with cros_test_lib.LoggingCapturer():
 239       self.rc_mock.AddCmdResult(partial_mock.ListRegex('-- true'),
 240                                 returncode=5)
 241       returncode = self.cmd_mock.inst.Run()
 242       self.assertEquals(returncode, 5)
 243
 244   def testLocalSDKPath(self):
 245     """Fetch components from a local --sdk-path."""
 246     sdk_dir = os.path.join(self.tempdir, 'sdk_dir')
 247     osutils.SafeMakedirs(sdk_dir)
 248     osutils.WriteFile(os.path.join(sdk_dir, constants.METADATA_JSON),
 249                       SDKFetcherMock.FAKE_METADATA)
 250     self.SetupCommandMock(extra_args=['--sdk-path', sdk_dir])
 251     with cros_test_lib.LoggingCapturer():
 252       self.cmd_mock.inst.Run()
 253
 254   def testGomaError(self):
 255     """We print an error message when GomaError is raised."""
 256     self.SetupCommandMock()
 257     with cros_test_lib.LoggingCapturer() as logs:
 258       self.PatchObject(cros_chrome_sdk.ChromeSDKCommand, '_FetchGoma',
 259                        side_effect=cros_chrome_sdk.GomaError())
 260       self.cmd_mock.inst.Run()
 261       self.AssertLogsContain(logs, 'Goma:')
 262
 263   def testSpecificComponent(self):
 264     """Tests that SDKFetcher.Prepare() handles |components| param properly."""
 265     sdk = cros_chrome_sdk.SDKFetcher(os.path.join(self.tempdir),
 266                                     SDKFetcherMock.BOARD)
 267     components = [constants.BASE_IMAGE_TAR, constants.CHROME_SYSROOT_TAR]
 268     with sdk.Prepare(components=components) as ctx:
 269       for c in components:
 270         self.assertTrue(os.path.exists(ctx.key_map[c].path))
 271       for c in [constants.IMAGE_SCRIPTS_TAR, constants.CHROME_ENV_TAR]:
 272         self.assertFalse(c in ctx.key_map)
 273
 274   @staticmethod
 275   def FindInPath(paths, endswith):
 276     for path in paths.split(':'):
 277       if path.endswith(endswith):
 278         return True
 279     return False
 280
 281   def testGomaInPath(self, inverted=False):
 282     """Verify that we do indeed add Goma to the PATH."""
 283     extra_args = ['--nogoma'] if inverted else None
 284     self.SetupCommandMock(extra_args)
 285     self.cmd_mock.inst.Run()
 286
 287     assert_fn = self.assertNotIn if inverted else self.assertIn
 288     gyp_defines_str = self.cmd_mock.env['GYP_DEFINES']
 289     gyp_defines = chrome_util.ProcessGypDefines(gyp_defines_str)
 290     assert_fn('gomadir', gyp_defines)
 291     assert_fn('use_goma', gyp_defines)
 292
 293   def testNoGoma(self):
 294     """Verify that we do not add Goma to the PATH."""
 295     self.testGomaInPath(inverted=True)
 296
 297   def testClang(self):
 298     """Verifies clang codepath."""
 299     with cros_test_lib.LoggingCapturer() as logs:
 300       cmd_cls = cros_chrome_sdk.ChromeSDKCommand
 301       update_sh = os.path.join(self.chrome_src_dir, cmd_cls._CLANG_UPDATE_SH)
 302       osutils.Touch(update_sh, makedirs=True)
 303       self.rc_mock.AddCmdResult(partial_mock.ListRegex('.*-gcc -dumpversion'),
 304                                 output='4.7.3')
 305       self.SetupCommandMock(extra_args=['--clang', '--make'])
 306       self.cmd_mock.inst.Run()
 307       self.assertTrue(self.FindInPath(
 308         self.cmd_mock.env['PATH'], cmd_cls._CLANG_DIR))
 309       self.AssertLogsContain(logs, '%s not found.' % cmd_cls._CLANG_DIR)
 310
 311
 312 class GomaTest(cros_test_lib.MockTempDirTestCase,
 313                cros_test_lib.LoggingTestCase):
 314   """Test Goma setup functionality."""
 315
 316   def setUp(self):
 317     self.rc_mock = cros_build_lib_unittest.RunCommandMock()
 318     self.rc_mock.SetDefaultCmdResult()
 319     self.StartPatcher(self.rc_mock)
 320
 321     self.cmd_mock = MockChromeSDKCommand(
 322         ['--board', SDKFetcherMock.BOARD, 'true'],
 323         base_args=['--cache-dir', self.tempdir])
 324     self.StartPatcher(self.cmd_mock)
 325
 326   def VerifyGomaError(self):
 327     self.assertRaises(cros_chrome_sdk.GomaError, self.cmd_mock.inst._FetchGoma)
 328
 329   def testNoGomaPort(self):
 330     """We print an error when gomacc is not returning a port."""
 331     self.rc_mock.AddCmdResult(
 332           cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD)
 333     self.VerifyGomaError()
 334
 335   def testGomaccError(self):
 336     """We print an error when gomacc exits with nonzero returncode."""
 337     self.rc_mock.AddCmdResult(
 338         cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD, returncode=1)
 339     self.VerifyGomaError()
 340
 341   def testFetchError(self):
 342     """We print an error when we can't fetch Goma."""
 343     self.rc_mock.AddCmdResult(
 344         cros_chrome_sdk.ChromeSDKCommand.GOMACC_PORT_CMD, returncode=1)
 345     self.VerifyGomaError()
 346
 347   def testGomaStart(self):
 348     """Test that we start Goma if it's not already started."""
 349     # Duplicate return values.
 350     self.PatchObject(cros_chrome_sdk.ChromeSDKCommand, '_GomaPort',
 351                      side_effect=['XXXX', 'XXXX'])
 352     # Run it twice to exercise caching.
 353     for _ in range(2):
 354       goma_dir, goma_port = self.cmd_mock.inst._FetchGoma()
 355       self.assertEquals(goma_port, 'XXXX')
 356       self.assertTrue(bool(goma_dir))
 357
 358
 359 class VersionTest(cros_test_lib.MockTempDirTestCase):
 360   """Tests the determination of which SDK version to use."""
 361
 362   VERSION = '3543.2.0'
 363   FULL_VERSION = 'R55-%s' % VERSION
 364   BOARD = 'lumpy'
 365
 366   VERSION_BASE = ('gs://chromeos-image-archive/%s-release/LATEST-%s'
 367                   % (BOARD, VERSION))
 368
 369   CAT_ERROR = ('InvalidUriError: Attempt to get key for '
 370                '%s failed.' % VERSION_BASE)
 371   LS_ERROR = 'CommandException: One or more URIs matched no objects.'
 372
 373   def setUp(self):
 374     self.gs_mock = self.StartPatcher(gs_unittest.GSContextMock())
 375     self.gs_mock.SetDefaultCmdResult()
 376     self.sdk_mock = self.StartPatcher(SDKFetcherMock(
 377         external_mocks=[self.gs_mock]))
 378
 379     os.environ.pop(cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV, None)
 380     self.sdk = cros_chrome_sdk.SDKFetcher(
 381         os.path.join(self.tempdir, 'cache'), self.BOARD)
 382
 383   def SetUpDefaultVersion(self, current, target, newest):
 384     self.PatchObject(cros_chrome_sdk.SDKFetcher, 'GetDefaultVersion',
 385                      return_value=current)
 386     self.PatchObject(cros_chrome_sdk.SDKFetcher, '_GetRepoCheckoutVersion',
 387                      return_value=target)
 388     self.PatchObject(cros_chrome_sdk.SDKFetcher, '_GetNewestManifestVersion',
 389                      return_value=newest)
 390     return self.sdk.UpdateDefaultVersion()
 391
 392   def testUpdateDefaultVersionNormal(self):
 393     """Updating default version with no cached default version."""
 394     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 395     target, updated = self.SetUpDefaultVersion(None, self.VERSION, '3544.0.0')
 396     self.assertEquals(target, self.VERSION)
 397     self.assertEquals(updated, True)
 398
 399   def testUpdateDefaultVersionTooNew(self):
 400     """Version in chromeos_version.sh isn't uploaded yet."""
 401     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 402     target, updated = self.SetUpDefaultVersion(None, '3543.10.0', self.VERSION)
 403     self.assertEquals(target, self.VERSION)
 404     self.assertEquals(updated, True)
 405
 406   def testUpdateDefaultVersionNoUpdate(self):
 407     """Nothing to update because the target version did not change."""
 408     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 409     target, updated = self.SetUpDefaultVersion(self.VERSION, self.VERSION,
 410                                                None)
 411     self.assertEquals(target, self.VERSION)
 412     self.assertEquals(updated, False)
 413
 414   def testUpdateDefaultChromeVersion(self):
 415     """We pick up the right LKGM version from the Chrome tree."""
 416     dir_struct = [
 417         'gclient_root/.gclient'
 418     ]
 419     cros_test_lib.CreateOnDiskHierarchy(self.tempdir, dir_struct)
 420     gclient_root = os.path.join(self.tempdir, 'gclient_root')
 421     self.PatchObject(os, 'getcwd', return_value=gclient_root)
 422
 423     lkgm_file = os.path.join(gclient_root, 'src', constants.PATH_TO_CHROME_LKGM)
 424     osutils.Touch(lkgm_file, makedirs=True)
 425     osutils.WriteFile(lkgm_file, self.VERSION)
 426     self.sdk_mock.UnMockAttr('UpdateDefaultVersion')
 427     self.sdk.UpdateDefaultVersion()
 428     self.assertEquals(self.sdk.GetDefaultVersion(),
 429                       self.VERSION)
 430
 431   def testFullVersion(self):
 432     """Test full version calculation."""
 433     def RaiseException(*_args, **_kwargs):
 434       raise Exception('boom')
 435
 436     self.sdk_mock.UnMockAttr('GetFullVersion')
 437     self.gs_mock.AddCmdResult(
 438         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 439         output=self.FULL_VERSION)
 440     self.assertEquals(
 441         self.FULL_VERSION,
 442         self.sdk.GetFullVersion(self.VERSION))
 443     # Test that we access the cache on the next call, rather than checking GS.
 444     self.gs_mock.AddCmdResult(
 445         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 446         side_effect=RaiseException)
 447     self.assertEquals(
 448         self.FULL_VERSION,
 449         self.sdk.GetFullVersion(self.VERSION))
 450
 451   def testBadVersion(self):
 452     """We raise an exception for a bad version."""
 453     self.sdk_mock.UnMockAttr('GetFullVersion')
 454     self.gs_mock.AddCmdResult(
 455         partial_mock.ListRegex('cat .*/LATEST-%s' % self.VERSION),
 456         output='', error=self.CAT_ERROR, returncode=1)
 457     self.gs_mock.AddCmdResult(
 458         partial_mock.ListRegex('ls .*%s' % self.VERSION),
 459         output='', error=self.LS_ERROR, returncode=1)
 460     self.assertRaises(cros_chrome_sdk.MissingSDK, self.sdk.GetFullVersion,
 461                       self.VERSION)
 462
 463   def testDefaultEnvBadBoard(self):
 464     """We don't use the version in the environment if board doesn't match."""
 465     os.environ[cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV] = self.VERSION
 466     self.assertNotEquals(self.VERSION, self.sdk_mock.VERSION)
 467     self.assertEquals(self.sdk.GetDefaultVersion(), None)
 468
 469   def testDefaultEnvGoodBoard(self):
 470     """We use the version in the environment if board matches."""
 471     sdk_version_env = cros_chrome_sdk.SDKFetcher.SDK_VERSION_ENV
 472     os.environ[sdk_version_env] = self.VERSION
 473     os.environ[cros_chrome_sdk.SDKFetcher.SDK_BOARD_ENV] = self.BOARD
 474     self.assertEquals(self.sdk.GetDefaultVersion(), self.VERSION)
 475
 476
 477 class PathVerifyTest(cros_test_lib.MockTempDirTestCase,
 478                      cros_test_lib.LoggingTestCase):
 479   """Tests user_rc PATH validation and warnings."""
 480
 481   def testPathVerifyWarnings(self):
 482     """Test the user rc PATH verification codepath."""
 483     def SourceEnvironmentMock(*_args, **_kwargs):
 484       return {
 485           'PATH': ':'.join([os.path.dirname(p) for p in abs_paths]),
 486       }
 487
 488     self.PatchObject(osutils, 'SourceEnvironment',
 489                      side_effect=SourceEnvironmentMock)
 490     file_list = (
 491       'goma/goma_ctl.py',
 492       'clang/clang',
 493       'chromite/parallel_emerge',
 494     )
 495     abs_paths = [os.path.join(self.tempdir, relpath) for relpath in file_list]
 496     for p in abs_paths:
 497       osutils.Touch(p, makedirs=True, mode=0o755)
 498
 499     with cros_test_lib.LoggingCapturer() as logs:
 500       cros_chrome_sdk.ChromeSDKCommand._VerifyGoma(None)
 501       cros_chrome_sdk.ChromeSDKCommand._VerifyClang(None)
 502       cros_chrome_sdk.ChromeSDKCommand._VerifyChromiteBin(None)
 503
 504     for msg in ['managed Goma', 'default Clang', 'default Chromite']:
 505       self.AssertLogsMatch(logs, msg)
 506
 507
 508 if __name__ == '__main__':
 509   cros_test_lib.main()