src/third_party/chromite/scripts/upload_symbols_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Unittests for upload_symbols.py"""
   7
   8 from __future__ import print_function
   9
  10 import BaseHTTPServer
  11 import ctypes
  12 import errno
  13 import logging
  14 import multiprocessing
  15 import os
  16 import signal
  17 import socket
  18 import SocketServer
  19 import sys
  20 import time
  21 import urllib2
  22
  23 sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)),
  24                                 '..', '..'))
  25 from chromite.lib import cros_build_lib
  26 from chromite.lib import cros_test_lib
  27 from chromite.lib import osutils
  28 from chromite.lib import parallel
  29 from chromite.lib import parallel_unittest
  30 from chromite.lib import remote_access
  31 from chromite.scripts import cros_generate_breakpad_symbols
  32 from chromite.scripts import upload_symbols
  33
  34 # TODO(build): Finish test wrapper (http://crosbug.com/37517).
  35 # Until then, this has to be after the chromite imports.
  36 import isolateserver
  37 import mock
  38
  39
  40 class SymbolServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
  41   """HTTP handler for symbol POSTs"""
  42
  43   RESP_CODE = None
  44   RESP_MSG = None
  45
  46   def do_POST(self):
  47     """Handle a POST request"""
  48     # Drain the data from the client.  If we don't, we might write the response
  49     # and close the socket before the client finishes, so they die with EPIPE.
  50     clen = int(self.headers.get('Content-Length', '0'))
  51     self.rfile.read(clen)
  52
  53     self.send_response(self.RESP_CODE, self.RESP_MSG)
  54     self.end_headers()
  55
  56   def log_message(self, *args, **kwargs):
  57     """Stub the logger as it writes to stderr"""
  58     pass
  59
  60
  61 class SymbolServer(SocketServer.ThreadingTCPServer, BaseHTTPServer.HTTPServer):
  62   """Simple HTTP server that forks each request"""
  63
  64
  65 class UploadSymbolsServerTest(cros_test_lib.MockTempDirTestCase):
  66   """Tests for UploadSymbols() and a local HTTP server"""
  67
  68   SYM_CONTENTS = """MODULE Linux arm 123-456 blkid
  69 PUBLIC 1471 0 main"""
  70
  71   def SpawnServer(self, RequestHandler):
  72     """Spawn a new http server"""
  73     while True:
  74       try:
  75         port = remote_access.GetUnusedPort()
  76         address = ('', port)
  77         self.httpd = SymbolServer(address, RequestHandler)
  78         break
  79       except socket.error as e:
  80         if e.errno == errno.EADDRINUSE:
  81           continue
  82         raise
  83     self.server = 'http://localhost:%i' % port
  84     self.httpd_pid = os.fork()
  85     if self.httpd_pid == 0:
  86       self.httpd.serve_forever(poll_interval=0.1)
  87       sys.exit(0)
  88
  89   def setUp(self):
  90     self.httpd_pid = None
  91     self.httpd = None
  92     self.server = None
  93     self.sym_file = os.path.join(self.tempdir, 'test.sym')
  94     osutils.WriteFile(self.sym_file, self.SYM_CONTENTS)
  95
  96   def tearDown(self):
  97     # Only kill the server if we forked one.
  98     if self.httpd_pid:
  99       os.kill(self.httpd_pid, signal.SIGUSR1)
 100
 101   def testSuccess(self):
 102     """The server returns success for all uploads"""
 103     class Handler(SymbolServerRequestHandler):
 104       """Always return 200"""
 105       RESP_CODE = 200
 106
 107     self.SpawnServer(Handler)
 108     ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0,
 109                                        sym_paths=[self.sym_file] * 10,
 110                                        retry=False)
 111     self.assertEqual(ret, 0)
 112
 113   def testError(self):
 114     """The server returns errors for all uploads"""
 115     class Handler(SymbolServerRequestHandler):
 116       """Always return 500"""
 117       RESP_CODE = 500
 118       RESP_MSG = 'Internal Server Error'
 119
 120     self.SpawnServer(Handler)
 121     ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0,
 122                                        sym_paths=[self.sym_file] * 10,
 123                                        retry=False)
 124     self.assertEqual(ret, 4)
 125
 126   def testHungServer(self):
 127     """The server chokes, but we recover"""
 128     class Handler(SymbolServerRequestHandler):
 129       """All connections choke forever"""
 130       def do_POST(self):
 131         while True:
 132           time.sleep(1000)
 133
 134     self.SpawnServer(Handler)
 135     with mock.patch.object(upload_symbols, 'GetUploadTimeout') as m:
 136       m.return_value = 0.1
 137       ret = upload_symbols.UploadSymbols('', server=self.server, sleep=0,
 138                                          sym_paths=[self.sym_file] * 10,
 139                                          retry=False)
 140     self.assertEqual(ret, 4)
 141
 142
 143 class UploadSymbolsTest(cros_test_lib.MockTempDirTestCase):
 144   """Tests for UploadSymbols()"""
 145
 146   def setUp(self):
 147     for d in ('foo', 'bar', 'some/dir/here'):
 148       d = os.path.join(self.tempdir, d)
 149       osutils.SafeMakedirs(d)
 150       for f in ('ignored', 'real.sym', 'no.sym.here'):
 151         f = os.path.join(d, f)
 152         osutils.Touch(f)
 153     self.sym_paths = [
 154         'bar/real.sym',
 155         'foo/real.sym',
 156         'some/dir/here/real.sym',
 157     ]
 158
 159     self.upload_mock = self.PatchObject(upload_symbols, 'UploadSymbol')
 160     self.PatchObject(cros_generate_breakpad_symbols, 'ReadSymsHeader',
 161                      return_value=cros_generate_breakpad_symbols.SymbolHeader(
 162                          os='os', cpu='cpu', id='id', name='name'))
 163
 164   def _testUploadURL(self, official, expected_url):
 165     """Helper for checking the url used"""
 166     self.upload_mock.return_value = 0
 167     with parallel_unittest.ParallelMock():
 168       ret = upload_symbols.UploadSymbols('', official=official, retry=False,
 169                                          breakpad_dir=self.tempdir, sleep=0)
 170       self.assertEqual(ret, 0)
 171       self.assertEqual(self.upload_mock.call_count, 3)
 172       for call_args in self.upload_mock.call_args_list:
 173         url, sym_item = call_args[0]
 174         self.assertEqual(url, expected_url)
 175         self.assertTrue(sym_item.sym_file.endswith('.sym'))
 176
 177   def testOfficialUploadURL(self):
 178     """Verify we upload to the real crash server for official builds"""
 179     self._testUploadURL(True, upload_symbols.OFFICIAL_UPLOAD_URL)
 180
 181   def testUnofficialUploadURL(self):
 182     """Verify we upload to the staging crash server for unofficial builds"""
 183     self._testUploadURL(False, upload_symbols.STAGING_UPLOAD_URL)
 184
 185   def testUploadSymbolFailureSimple(self):
 186     """Verify that when UploadSymbol fails, the error count is passed up"""
 187     def UploadSymbol(*_args, **kwargs):
 188       kwargs['num_errors'].value = 4
 189     self.upload_mock.side_effect = UploadSymbol
 190     with parallel_unittest.ParallelMock():
 191       ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir, sleep=0,
 192                                          retry=False)
 193       self.assertEquals(ret, 4)
 194
 195   def testUploadCount(self):
 196     """Verify we can limit the number of uploaded symbols"""
 197     self.upload_mock.return_value = 0
 198     for c in xrange(3):
 199       self.upload_mock.reset_mock()
 200       with parallel_unittest.ParallelMock():
 201         ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir,
 202                                            sleep=0, upload_limit=c)
 203         self.assertEquals(ret, 0)
 204         self.assertEqual(self.upload_mock.call_count, c)
 205
 206   def testFailedFileList(self):
 207     """Verify the failed file list is populated with the right content"""
 208     def UploadSymbol(*args, **kwargs):
 209       kwargs['failed_queue'].put(args[1].sym_file)
 210       kwargs['num_errors'].value = 4
 211     self.upload_mock.side_effect = UploadSymbol
 212     with parallel_unittest.ParallelMock():
 213       failed_list = os.path.join(self.tempdir, 'list')
 214       ret = upload_symbols.UploadSymbols('', breakpad_dir=self.tempdir, sleep=0,
 215                                          retry=False, failed_list=failed_list)
 216       self.assertEquals(ret, 4)
 217
 218       # Need to sort the output as parallel/fs discovery can be unordered.
 219       got_list = sorted(osutils.ReadFile(failed_list).splitlines())
 220       self.assertEquals(self.sym_paths, got_list)
 221
 222   def _testUpload(self, inputs, sym_paths=None):
 223     """Helper for testing uploading of specific paths"""
 224     if sym_paths is None:
 225       sym_paths = inputs
 226
 227     self.upload_mock.return_value = 0
 228     with parallel_unittest.ParallelMock():
 229       ret = upload_symbols.UploadSymbols(sym_paths=inputs, sleep=0,
 230                                          retry=False)
 231       self.assertEquals(ret, 0)
 232       self.assertEquals(self.upload_mock.call_count, len(sym_paths))
 233
 234       # Since upload order is arbitrary, we have to do a manual scan for each
 235       # path ourselves against the uploaded file list.
 236       found_syms = [x[0][1].sym_file for x in self.upload_mock.call_args_list]
 237       for found_sym in found_syms:
 238         for path in sym_paths:
 239           if found_sym.endswith(path):
 240             break
 241         else:
 242           raise AssertionError('Could not locate %s in %r' % (path, found_syms))
 243
 244   def testUploadFiles(self):
 245     """Test uploading specific symbol files"""
 246     sym_paths = (
 247         os.path.join(self.tempdir, 'bar', 'real.sym'),
 248         os.path.join(self.tempdir, 'foo', 'real.sym'),
 249     )
 250     self._testUpload(sym_paths)
 251
 252   def testUploadDirectory(self):
 253     """Test uploading directory of symbol files"""
 254     self._testUpload([self.tempdir], sym_paths=self.sym_paths)
 255
 256   def testUploadLocalTarball(self):
 257     """Test uploading symbols contains in a local tarball"""
 258     tarball = os.path.join(self.tempdir, 'syms.tar.gz')
 259     cros_build_lib.CreateTarball(
 260         'syms.tar.gz', self.tempdir, compression=cros_build_lib.COMP_GZIP,
 261         inputs=('foo', 'bar', 'some'))
 262     self._testUpload([tarball], sym_paths=self.sym_paths)
 263
 264   def testUploadRemoteTarball(self):
 265     """Test uploading symbols contains in a remote tarball"""
 266     # TODO: Need to figure out how to mock out lib.cache.TarballCache.
 267
 268   def testDedupeNotifyFailure(self):
 269     """Test that a dedupe server failure midway doesn't wedge things"""
 270     api_mock = mock.MagicMock()
 271
 272     def _Contains(items):
 273       """Do not dedupe anything"""
 274       return items
 275     api_mock.contains.side_effect = _Contains
 276
 277     # Use a list so the closure below can modify the value.
 278     item_count = [0]
 279     # Pick a number big enough to trigger a hang normally, but not so
 280     # big it adds a lot of overhead.
 281     item_limit = 50
 282     def _Push(*_args):
 283       """Die in the middle of the push list"""
 284       item_count[0] += 1
 285       if item_count[0] > (item_limit / 10):
 286         raise ValueError('time to die')
 287     api_mock.push.side_effect = _Push
 288
 289     self.PatchObject(isolateserver, 'get_storage_api', return_value=api_mock)
 290
 291     def _Uploader(*args, **kwargs):
 292       """Pass the uploaded symbol to the deduper"""
 293       sym_item = args[1]
 294       passed_queue = kwargs['passed_queue']
 295       passed_queue.put(sym_item)
 296     self.upload_mock.side_effect = _Uploader
 297
 298     self.upload_mock.return_value = 0
 299     with parallel_unittest.ParallelMock():
 300       ret = upload_symbols.UploadSymbols(
 301           '', sym_paths=[self.tempdir] * item_limit, sleep=0,
 302           dedupe_namespace='inva!id name$pace')
 303       self.assertEqual(ret, 0)
 304       # This test normally passes by not hanging.
 305
 306   def testSlowDedupeSystem(self):
 307     """Verify a slow-to-join process doesn't break things when dedupe is off"""
 308     # The sleep value here is inherently a little racy, but seems to be good
 309     # enough to trigger the bug on a semi-regular basis on developer systems.
 310     self.PatchObject(upload_symbols, 'SymbolDeduplicatorNotify',
 311                      side_effect=lambda *args: time.sleep(1))
 312     # Test passing means the code didn't throw an exception.
 313     upload_symbols.UploadSymbols(sym_paths=[self.tempdir])
 314
 315
 316 class SymbolDeduplicatorNotifyTest(cros_test_lib.MockTestCase):
 317   """Tests for SymbolDeduplicatorNotify()"""
 318
 319   def setUp(self):
 320     self.storage_mock = self.PatchObject(isolateserver, 'get_storage_api')
 321
 322   def testSmoke(self):
 323     """Basic run through the system."""
 324     q = mock.MagicMock()
 325     q.get.side_effect = (upload_symbols.FakeItem(), None,)
 326     upload_symbols.SymbolDeduplicatorNotify('name', q)
 327
 328   def testStorageException(self):
 329     """We want to just warn & move on when dedupe server fails"""
 330     log_mock = self.PatchObject(cros_build_lib, 'Warning')
 331     q = mock.MagicMock()
 332     q.get.side_effect = (upload_symbols.FakeItem(), None,)
 333     self.storage_mock.side_effect = Exception
 334     upload_symbols.SymbolDeduplicatorNotify('name', q)
 335     self.assertEqual(log_mock.call_count, 1)
 336
 337
 338 class SymbolDeduplicatorTest(cros_test_lib.MockTestCase):
 339   """Tests for SymbolDeduplicator()"""
 340
 341   def setUp(self):
 342     self.storage_mock = mock.MagicMock()
 343     self.header_mock = self.PatchObject(
 344         cros_generate_breakpad_symbols, 'ReadSymsHeader',
 345         return_value=cros_generate_breakpad_symbols.SymbolHeader(
 346             os='os', cpu='cpu', id='id', name='name'))
 347
 348   def testNoStorageOrPaths(self):
 349     """We don't want to talk to the server if there's no storage or files"""
 350     upload_symbols.SymbolDeduplicator(None, [])
 351     upload_symbols.SymbolDeduplicator(self.storage_mock, [])
 352     self.assertEqual(self.storage_mock.call_count, 0)
 353     self.assertEqual(self.header_mock.call_count, 0)
 354
 355   def testStorageException(self):
 356     """We want to just warn & move on when dedupe server fails"""
 357     log_mock = self.PatchObject(cros_build_lib, 'Warning')
 358     self.storage_mock.contains.side_effect = Exception('storage error')
 359     sym_paths = ['/a', '/bbbbbb', '/cc.c']
 360     ret = upload_symbols.SymbolDeduplicator(self.storage_mock, sym_paths)
 361     self.assertEqual(log_mock.call_count, 1)
 362     self.assertEqual(self.storage_mock.contains.call_count, 1)
 363     self.assertEqual(self.header_mock.call_count, len(sym_paths))
 364     self.assertEqual(len(ret), len(sym_paths))
 365
 366
 367 class UploadSymbolTest(cros_test_lib.MockTempDirTestCase):
 368   """Tests for UploadSymbol()"""
 369
 370   def setUp(self):
 371     self.sym_file = os.path.join(self.tempdir, 'foo.sym')
 372     self.sym_item = upload_symbols.FakeItem(sym_file=self.sym_file)
 373     self.url = 'http://eatit'
 374     self.upload_mock = self.PatchObject(upload_symbols, 'SymUpload')
 375
 376   def testUploadSymbolNormal(self):
 377     """Verify we try to upload on a normal file"""
 378     osutils.Touch(self.sym_file)
 379     ret = upload_symbols.UploadSymbol(self.url, self.sym_item)
 380     self.assertEqual(ret, 0)
 381     self.upload_mock.assert_called_with(self.url, self.sym_item)
 382     self.assertEqual(self.upload_mock.call_count, 1)
 383
 384   def testUploadSymbolErrorCountExceeded(self):
 385     """Verify that when the error count gets too high, we stop uploading"""
 386     errors = ctypes.c_int(10000)
 387     # Pass in garbage values so that we crash if num_errors isn't handled.
 388     ret = upload_symbols.UploadSymbol(None, self.sym_item, sleep=None,
 389                                       num_errors=errors)
 390     self.assertEqual(ret, 0)
 391
 392   def testUploadRetryErrors(self, side_effect=None):
 393     """Verify that we retry errors (and eventually give up)"""
 394     if not side_effect:
 395       side_effect = urllib2.HTTPError('http://', 400, 'fail', {}, None)
 396     self.upload_mock.side_effect = side_effect
 397     errors = ctypes.c_int()
 398     item = upload_symbols.FakeItem(sym_file='/dev/null')
 399     ret = upload_symbols.UploadSymbol(self.url, item, num_errors=errors)
 400     self.assertEqual(ret, 1)
 401     self.upload_mock.assert_called_with(self.url, item)
 402     self.assertTrue(self.upload_mock.call_count >= upload_symbols.MAX_RETRIES)
 403
 404   def testConnectRetryErrors(self):
 405     """Verify that we retry errors (and eventually give up) w/connect errors"""
 406     side_effect = urllib2.URLError('foo')
 407     self.testUploadRetryErrors(side_effect=side_effect)
 408
 409   def testTruncateTooBigFiles(self):
 410     """Verify we shrink big files"""
 411     def SymUpload(_url, sym_item):
 412       content = osutils.ReadFile(sym_item.sym_file)
 413       self.assertEqual(content, 'some junk\n')
 414     self.upload_mock.upload_mock.side_effect = SymUpload
 415     content = '\n'.join((
 416         'STACK CFI 1234',
 417         'some junk',
 418         'STACK CFI 1234',
 419     ))
 420     osutils.WriteFile(self.sym_file, content)
 421     ret = upload_symbols.UploadSymbol(self.url, self.sym_item, file_limit=1)
 422     self.assertEqual(ret, 0)
 423     # Make sure the item passed to the upload has a temp file and not the
 424     # original -- only the temp one has been stripped down.
 425     temp_item = self.upload_mock.call_args[0][1]
 426     self.assertNotEqual(temp_item.sym_file, self.sym_item.sym_file)
 427     self.assertEqual(self.upload_mock.call_count, 1)
 428
 429   def testTruncateReallyLargeFiles(self):
 430     """Verify we try to shrink really big files"""
 431     warn_mock = self.PatchObject(cros_build_lib, 'PrintBuildbotStepWarnings')
 432     with open(self.sym_file, 'w+b') as f:
 433       f.truncate(upload_symbols.CRASH_SERVER_FILE_LIMIT + 100)
 434       f.seek(0)
 435       f.write('STACK CFI 1234\n\n')
 436     ret = upload_symbols.UploadSymbol(self.url, self.sym_item)
 437     self.assertEqual(ret, 0)
 438     # Make sure the item passed to the upload has a temp file and not the
 439     # original -- only the temp one has been truncated.
 440     temp_item = self.upload_mock.call_args[0][1]
 441     self.assertNotEqual(temp_item.sym_file, self.sym_item.sym_file)
 442     self.assertEqual(self.upload_mock.call_count, 1)
 443     self.assertEqual(warn_mock.call_count, 1)
 444
 445
 446 class SymUploadTest(cros_test_lib.MockTempDirTestCase):
 447   """Tests for SymUpload()"""
 448
 449   SYM_URL = 'http://localhost/post/it/here'
 450   SYM_CONTENTS = """MODULE Linux arm 123-456 blkid
 451 PUBLIC 1471 0 main"""
 452
 453   def setUp(self):
 454     self.sym_file = os.path.join(self.tempdir, 'test.sym')
 455     osutils.WriteFile(self.sym_file, self.SYM_CONTENTS)
 456     self.sym_item = upload_symbols.SymbolItem(self.sym_file)
 457
 458   def testPostUpload(self):
 459     """Verify HTTP POST has all the fields we need"""
 460     m = self.PatchObject(urllib2, 'urlopen', autospec=True)
 461     upload_symbols.SymUpload(self.SYM_URL, self.sym_item)
 462     self.assertEquals(m.call_count, 1)
 463     req = m.call_args[0][0]
 464     self.assertEquals(req.get_full_url(), self.SYM_URL)
 465     data = ''.join([x for x in req.get_data()])
 466
 467     fields = {
 468         'code_file': 'blkid',
 469         'debug_file': 'blkid',
 470         'debug_identifier': '123456',
 471         'os': 'Linux',
 472         'cpu': 'arm',
 473     }
 474     for key, val in fields.iteritems():
 475       line = 'Content-Disposition: form-data; name="%s"\r\n' % key
 476       self.assertTrue(line in data)
 477       line = '%s\r\n' % val
 478       self.assertTrue(line in data)
 479     line = ('Content-Disposition: form-data; name="symbol_file"; '
 480             'filename="test.sym"\r\n')
 481     self.assertTrue(line in data)
 482     self.assertTrue(self.SYM_CONTENTS in data)
 483
 484   def testTimeout(self):
 485     """Verify timeouts scale based on filesize"""
 486     m = self.PatchObject(urllib2, 'urlopen', autospec=True)
 487     size = self.PatchObject(os.path, 'getsize')
 488
 489     tests = (
 490         # Small files should get rounded up to the minimum timeout.
 491         (10, upload_symbols.UPLOAD_MIN_TIMEOUT),
 492         # A 50MiB file should take like ~4 minutes.
 493         (50 * 1024 * 1024, 257),
 494     )
 495     for size.return_value, timeout in tests:
 496       upload_symbols.SymUpload(self.SYM_URL, self.sym_item)
 497       self.assertEqual(m.call_args[1]['timeout'], timeout)
 498
 499
 500 class UtilTest(cros_test_lib.TempDirTestCase):
 501   """Various tests for utility funcs."""
 502
 503   def testWriteQueueToFile(self):
 504     """Basic test for WriteQueueToFile."""
 505     listing = os.path.join(self.tempdir, 'list')
 506     exp_list = [
 507         'b/c.txt',
 508         'foo.log',
 509         'there/might/be/giants',
 510     ]
 511     relpath = '/a'
 512
 513     q = multiprocessing.Queue()
 514     for f in exp_list:
 515       q.put(os.path.join(relpath, f))
 516     q.put(None)
 517     upload_symbols.WriteQueueToFile(listing, q, '/a')
 518
 519     got_list = osutils.ReadFile(listing).splitlines()
 520     self.assertEquals(exp_list, got_list)
 521
 522
 523 if __name__ == '__main__':
 524   # pylint: disable=W0212
 525   # Set timeouts small so that if the unit test hangs, it won't hang for long.
 526   parallel._BackgroundTask.STARTUP_TIMEOUT = 5
 527   parallel._BackgroundTask.EXIT_TIMEOUT = 5
 528
 529   # We want to test retry behavior, so make sure we don't sleep.
 530   upload_symbols.INITIAL_RETRY_DELAY = 0
 531
 532   # Run the tests.
 533   cros_test_lib.main(level=logging.INFO)