1 # Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Upload all debug symbols required for crash reporting purposes.
7 This script need only be used to upload release builds symbols or to debug
8 crashes on non-release builds (in which case try to only upload the symbols
9 for those executables involved).
12 from __future__ import print_function
19 import multiprocessing
30 from chromite.buildbot import constants
31 from chromite.lib import cache
32 from chromite.lib import commandline
33 from chromite.lib import cros_build_lib
34 from chromite.lib import gs
35 from chromite.lib import osutils
36 from chromite.lib import parallel
37 from chromite.lib import retry_util
38 from chromite.lib import timeout_util
39 from chromite.scripts import cros_generate_breakpad_symbols
41 # Needs to be after chromite imports.
42 # TODO(build): When doing the initial buildbot bootstrap, we won't have any
43 # other repos available. So ignore isolateserver imports. But buildbot will
44 # re-exec itself once it has done a full repo sync and then the module will
45 # be available -- it isn't needed that early. http://crbug.com/341152
52 # URLs used for uploading symbols.
53 OFFICIAL_UPLOAD_URL = 'http://clients2.google.com/cr/symbol'
54 STAGING_UPLOAD_URL = 'http://clients2.google.com/cr/staging_symbol'
57 # The crash server rejects files that are this big.
58 CRASH_SERVER_FILE_LIMIT = 350 * 1024 * 1024
59 # Give ourselves a little breathing room from what the server expects.
60 DEFAULT_FILE_LIMIT = CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024)
63 # The batch limit when talking to the dedup server. We avoid sending one at a
64 # time as the round trip overhead will dominate. Conversely, we avoid sending
65 # all at once so we can start uploading symbols asap -- the symbol server is a
66 # bit slow and will take longer than anything else.
67 # TODO: A better algorithm would be adaptive. If we have more than one symbol
68 # in the upload queue waiting, we could send more symbols to the dedupe server
72 # How long to wait for the server to respond with the results. Note that the
73 # larger the limit above, the larger this will need to be. So we give it ~1
74 # second per item max.
75 DEDUPE_TIMEOUT = DEDUPE_LIMIT
77 # The unique namespace in the dedupe server that only we use. Helps avoid
78 # collisions with all the hashed values and unrelated content.
79 OFFICIAL_DEDUPE_NAMESPACE = 'chromium-os-upload-symbols'
80 STAGING_DEDUPE_NAMESPACE = '%s-staging' % OFFICIAL_DEDUPE_NAMESPACE
83 # How long to wait (in seconds) for a single upload to complete. This has
84 # to allow for symbols that are up to CRASH_SERVER_FILE_LIMIT in size.
85 UPLOAD_TIMEOUT = 30 * 60
88 # Sleep for 200ms in between uploads to avoid DoS'ing symbol server.
89 DEFAULT_SLEEP_DELAY = 0.2
92 # Number of seconds to wait before retrying an upload. The delay will double
93 # for each subsequent retry of the same symbol file.
94 INITIAL_RETRY_DELAY = 1
96 # Allow up to 7 attempts to upload a symbol file (total delay may be
97 # 1+2+4+8+16+32=63 seconds).
100 # Number of total errors, before uploads are no longer attempted.
101 # This is used to avoid lots of errors causing unreasonable delays.
102 # See the related, but independent, error values below.
103 MAX_TOTAL_ERRORS_FOR_RETRY = 30
105 # A watermark of transient errors which we allow recovery from. If we hit
106 # errors infrequently, overall we're probably doing fine. For example, if
107 # we have one failure every 100 passes, then we probably don't want to fail
108 # right away. But if we hit a string of failures in a row, we want to abort.
110 # The watermark starts at 0 (and can never go below that). When this error
111 # level is exceeded, we stop uploading. When a failure happens, we add the
112 # fail adjustment, and when an upload succeeds, we add the pass adjustment.
113 # We want to penalize failures more so that we ramp up when there is a string
114 # of them, but then slowly back off as things start working.
117 # 0.0: Starting point.
118 # 0.0: Upload works, so add -0.5, and then clamp to 0.
119 # 1.0: Upload fails, so add 1.0.
120 # 2.0: Upload fails, so add 1.0.
121 # 1.5: Upload works, so add -0.5.
122 # 1.0: Upload works, so add -0.5.
123 ERROR_WATERMARK = 3.0
124 ERROR_ADJUST_FAIL = 1.0
125 ERROR_ADJUST_PASS = -0.5
128 def SymUpload(upload_url, sym_item):
129 """Upload a symbol file to a HTTP server
131 The upload is a multipart/form-data POST with the following parameters:
132 code_file: the basename of the module, e.g. "app"
133 code_identifier: the module file's identifier
134 debug_file: the basename of the debugging file, e.g. "app"
135 debug_identifier: the debug file's identifier, usually consisting of
136 the guid and age embedded in the pdb, e.g.
137 "11111111BBBB3333DDDD555555555555F"
138 version: the file version of the module, e.g. "1.2.3.4"
139 product: HTTP-friendly product name
140 os: the operating system that the module was built for
141 cpu: the CPU that the module was built for
142 symbol_file: the contents of the breakpad-format symbol file
145 upload_url: The crash URL to POST the |sym_file| to
146 sym_item: A SymbolItem containing the path to the breakpad symbol to upload
148 sym_header = sym_item.sym_header
149 sym_file = sym_item.sym_file
152 ('code_file', sym_header.name),
153 ('debug_file', sym_header.name),
154 ('debug_identifier', sym_header.id.replace('-', '')),
155 # The product/version fields are used by the server only for statistic
156 # purposes. They do not impact symbolization, so they're safe to set
157 # to any value all the time.
158 # In this case, we use it to help see the load our build system is
159 # placing on the server.
160 # Not sure what to set for the version. Maybe the git sha1 of this file.
161 # Note: the server restricts this to 30 chars.
163 ('product', 'ChromeOS'),
164 ('os', sym_header.os),
165 ('cpu', sym_header.cpu),
166 poster.encode.MultipartParam.from_file('symbol_file', sym_file),
169 data, headers = poster.encode.multipart_encode(fields)
170 request = urllib2.Request(upload_url, data, headers)
171 request.add_header('User-agent', 'chromite.upload_symbols')
172 urllib2.urlopen(request, timeout=UPLOAD_TIMEOUT)
175 def TestingSymUpload(upload_url, sym_item):
176 """A stub version of SymUpload for --testing usage"""
177 cmd = ['sym_upload', sym_item.sym_file, upload_url]
178 # Randomly fail 80% of the time (the retry logic makes this 80%/3 per file).
179 returncode = random.randint(1, 100) <= 80
180 cros_build_lib.Debug('would run (and return %i): %s', returncode,
181 cros_build_lib.CmdToStr(cmd))
183 output = 'Failed to send the symbol file.'
185 output = 'Successfully sent the symbol file.'
186 result = cros_build_lib.CommandResult(cmd=cmd, error=None, output=output,
187 returncode=returncode)
190 socket.error('[socket.error] forced test fail'),
191 httplib.BadStatusLine('[BadStatusLine] forced test fail'),
192 urllib2.HTTPError(upload_url, 400, '[HTTPError] forced test fail',
194 urllib2.URLError('[URLError] forced test fail'),
196 raise random.choice(exceptions)
201 def ErrorLimitHit(num_errors, watermark_errors):
202 """See if our error limit has been hit
205 num_errors: A multiprocessing.Value of the raw number of failures.
206 watermark_errors: A multiprocessing.Value of the current rate of failures.
209 True if our error limits have been exceeded.
211 return ((num_errors is not None and
212 num_errors.value > MAX_TOTAL_ERRORS_FOR_RETRY) or
213 (watermark_errors is not None and
214 watermark_errors.value > ERROR_WATERMARK))
217 def _UpdateCounter(counter, adj):
218 """Update |counter| by |adj|
220 Handle atomic updates of |counter|. Also make sure it does not
224 counter: A multiprocessing.Value to update
225 adj: The value to add to |counter|
228 clamp = 0 if type(adj) is int else 0.0
229 counter.value = max(clamp, counter.value + adj)
231 if hasattr(counter, 'get_lock'):
232 with counter.get_lock():
234 elif counter is not None:
238 def UploadSymbol(upload_url, sym_item, file_limit=DEFAULT_FILE_LIMIT,
239 sleep=0, num_errors=None, watermark_errors=None,
240 failed_queue=None, passed_queue=None):
241 """Upload |sym_item| to |upload_url|
244 upload_url: The crash server to upload things to
245 sym_item: A SymbolItem containing the path to the breakpad symbol to upload
246 file_limit: The max file size of a symbol file before we try to strip it
247 sleep: Number of seconds to sleep before running
248 num_errors: An object to update with the error count (needs a .value member)
249 watermark_errors: An object to track current error behavior (needs a .value)
250 failed_queue: When a symbol fails, add it to this queue
251 passed_queue: When a symbol passes, add it to this queue
254 The number of errors that were encountered.
256 sym_file = sym_item.sym_file
257 upload_item = sym_item
259 if num_errors is None:
260 num_errors = ctypes.c_int()
261 if ErrorLimitHit(num_errors, watermark_errors):
262 # Abandon ship! It's on fire! NOoooooooooooOOOoooooo.
264 failed_queue.put(sym_file)
268 # Keeps us from DoS-ing the symbol server.
271 cros_build_lib.Debug('uploading %s' % sym_file)
273 # Ideally there'd be a tempfile.SpooledNamedTemporaryFile that we could use.
274 with tempfile.NamedTemporaryFile(prefix='upload_symbols',
275 bufsize=0) as temp_sym_file:
277 # If the symbols size is too big, strip out the call frame info. The CFI
278 # is unnecessary for 32bit x86 targets where the frame pointer is used (as
279 # all of ours have) and it accounts for over half the size of the symbols
281 file_size = os.path.getsize(sym_file)
282 if file_size > file_limit:
283 cros_build_lib.Warning('stripping CFI from %s due to size %s > %s',
284 sym_file, file_size, file_limit)
285 temp_sym_file.writelines([x for x in open(sym_file, 'rb').readlines()
286 if not x.startswith('STACK CFI')])
288 upload_item = FakeItem(sym_file=temp_sym_file.name,
289 sym_header=sym_item.sym_header)
291 # Hopefully the crash server will let it through. But it probably won't.
292 # Not sure what the best answer is in this case.
293 file_size = os.path.getsize(upload_item.sym_file)
294 if file_size > CRASH_SERVER_FILE_LIMIT:
295 cros_build_lib.PrintBuildbotStepWarnings()
296 cros_build_lib.Warning('upload file %s is awfully large, risking '
297 'rejection by the symbol server (%s > %s)',
298 sym_file, file_size, CRASH_SERVER_FILE_LIMIT)
300 # Upload the symbol file.
303 cros_build_lib.TimedCommand(
304 retry_util.RetryException,
305 (urllib2.HTTPError, urllib2.URLError), MAX_RETRIES, SymUpload,
306 upload_url, upload_item, sleep=INITIAL_RETRY_DELAY,
307 timed_log_msg='upload of %10i bytes took %%s: %s' %
308 (file_size, os.path.basename(sym_file)))
312 passed_queue.put(sym_item)
313 except urllib2.HTTPError as e:
314 cros_build_lib.Warning('could not upload: %s: HTTP %s: %s',
315 os.path.basename(sym_file), e.code, e.reason)
316 except (urllib2.URLError, httplib.HTTPException, socket.error) as e:
317 cros_build_lib.Warning('could not upload: %s: %s',
318 os.path.basename(sym_file), e)
321 _UpdateCounter(watermark_errors, ERROR_ADJUST_PASS)
323 _UpdateCounter(num_errors, 1)
324 _UpdateCounter(watermark_errors, ERROR_ADJUST_FAIL)
326 failed_queue.put(sym_file)
328 return num_errors.value
331 # A dummy class that allows for stubbing in tests and SymUpload.
332 FakeItem = cros_build_lib.Collection(
333 'FakeItem', sym_file=None, sym_header=None, content=lambda x: '')
336 # TODO(build): Delete this if check. http://crbug.com/341152
338 class SymbolItem(isolateserver.BufferItem):
339 """Turn a sym_file into an isolateserver.Item"""
343 def __init__(self, sym_file):
344 sym_header = cros_generate_breakpad_symbols.ReadSymsHeader(sym_file)
345 super(SymbolItem, self).__init__(str(sym_header), self.ALGO)
346 self.sym_header = sym_header
347 self.sym_file = sym_file
350 def SymbolDeduplicatorNotify(dedupe_namespace, dedupe_queue):
351 """Send a symbol file to the swarming service
353 Notify the swarming service of a successful upload. If the notification fails
354 for any reason, we ignore it. We don't care as it just means we'll upload it
355 again later on, and the symbol server will handle that graciously.
357 This func runs in a different process from the main one, so we cannot share
358 the storage object. Instead, we create our own. This func stays alive for
359 the life of the process, so we only create one here overall.
362 dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
363 dedupe_queue: The queue to read SymbolItems from
365 if dedupe_queue is None:
370 storage = isolateserver.get_storage_api(constants.ISOLATESERVER,
372 for item in iter(dedupe_queue.get, None):
373 with timeout_util.Timeout(DEDUPE_TIMEOUT):
374 cros_build_lib.Debug('sending %s to dedupe server', item.sym_file)
375 storage.push(item, item.content(0))
376 cros_build_lib.Info('dedupe notification finished; exiting')
378 sym_file = item.sym_file if (item and item.sym_file) else ''
379 cros_build_lib.Warning('posting %s to dedupe server failed',
380 os.path.basename(sym_file), exc_info=True)
382 # Keep draining the queue though so it doesn't fill up.
383 while dedupe_queue.get() is not None:
387 def SymbolDeduplicator(storage, sym_paths):
388 """Filter out symbol files that we've already uploaded
390 Using the swarming service, ask it to tell us which symbol files we've already
391 uploaded in previous runs and/or by other bots. If the query fails for any
392 reason, we'll just upload all symbols. This is fine as the symbol server will
393 do the right thing and this phase is purely an optimization.
395 This code runs in the main thread which is why we can re-use the existing
396 storage object. Saves us from having to recreate one all the time.
399 storage: An isolateserver.StorageApi object
400 sym_paths: List of symbol files to check against the dedupe server
403 List of symbol files that have not been uploaded before
408 items = [SymbolItem(x) for x in sym_paths]
411 with timeout_util.Timeout(DEDUPE_TIMEOUT):
412 items = storage.contains(items)
414 cros_build_lib.Warning('talking to dedupe server failed', exc_info=True)
420 """Guess if this is a tarball based on the filename."""
421 parts = path.split('.')
425 if parts[-1] == 'tar':
428 if parts[-2] == 'tar':
429 return parts[-1] in ('bz2', 'gz', 'xz')
431 return parts[-1] in ('tbz2', 'tbz', 'tgz', 'txz')
434 def SymbolFinder(tempdir, paths):
435 """Locate symbol files in |paths|
438 tempdir: Path to use for temporary files (caller will clean up).
439 paths: A list of input paths to walk. Files are returned w/out any checks.
440 Dirs are searched for files that end in ".sym". Urls are fetched and then
441 processed. Tarballs are unpacked and walked.
444 Yield every viable sym file.
447 # Pylint is confused about members of ParseResult.
449 o = urlparse.urlparse(p)
450 if o.scheme: # pylint: disable=E1101
451 # Support globs of filenames.
454 cros_build_lib.Info('processing files inside %s', p)
455 o = urlparse.urlparse(p)
456 cache_dir = commandline.GetCacheDir()
457 common_path = os.path.join(cache_dir, constants.COMMON_CACHE)
458 tar_cache = cache.TarballCache(common_path)
459 key = ('%s%s' % (o.netloc, o.path)).split('/') # pylint: disable=E1101
460 # The common cache will not be LRU, removing the need to hold a read
461 # lock on the cached gsutil.
462 ref = tar_cache.Lookup(key)
465 except cros_build_lib.RunCommandError as e:
466 cros_build_lib.Warning('ignoring %s\n%s', p, e)
468 for p in SymbolFinder(tempdir, [ref.path]):
471 elif os.path.isdir(p):
472 for root, _, files in os.walk(p):
474 if f.endswith('.sym'):
475 yield os.path.join(root, f)
478 cros_build_lib.Info('processing files inside %s', p)
479 tardir = tempfile.mkdtemp(dir=tempdir)
480 cache.Untar(os.path.realpath(p), tardir)
481 for p in SymbolFinder(tardir, [tardir]):
488 def WriteQueueToFile(listing, queue, relpath=None):
489 """Write all the items in |queue| to the |listing|.
491 Note: The queue must have a sentinel None appended to the end.
494 listing: Where to write out the list of files.
495 queue: The queue of paths to drain.
496 relpath: If set, write out paths relative to this one.
499 # Still drain the queue so we make sure the producer has finished
500 # before we return. Otherwise, the queue might get destroyed too
501 # quickly which will trigger a traceback in the producer.
502 while queue.get() is not None:
506 with cros_build_lib.Open(listing, 'wb+') as f:
512 path = os.path.relpath(path, relpath)
513 f.write('%s\n' % path)
516 def UploadSymbols(board=None, official=False, breakpad_dir=None,
517 file_limit=DEFAULT_FILE_LIMIT, sleep=DEFAULT_SLEEP_DELAY,
518 upload_limit=None, sym_paths=None, failed_list=None,
519 root=None, retry=True, dedupe_namespace=None):
520 """Upload all the generated symbols for |board| to the crash server
522 You can use in a few ways:
523 * pass |board| to locate all of its symbols
524 * pass |breakpad_dir| to upload all the symbols in there
525 * pass |sym_paths| to upload specific symbols (or dirs of symbols)
528 board: The board whose symbols we wish to upload
529 official: Use the official symbol server rather than the staging one
530 breakpad_dir: The full path to the breakpad directory where symbols live
531 file_limit: The max file size of a symbol file before we try to strip it
532 sleep: How long to sleep in between uploads
533 upload_limit: If set, only upload this many symbols (meant for testing)
534 sym_paths: Specific symbol files (or dirs of sym files) to upload,
535 otherwise search |breakpad_dir|
536 failed_list: Write the names of all sym files we did not upload; can be a
537 filename or file-like object.
538 root: The tree to prefix to |breakpad_dir| (if |breakpad_dir| is not set)
539 retry: Whether we should retry failures.
540 dedupe_namespace: The isolateserver namespace to dedupe uploaded symbols.
543 The number of errors that were encountered.
545 # TODO(build): Delete this assert.
546 assert isolateserver, 'Missing isolateserver import http://crbug.com/341152'
549 upload_url = OFFICIAL_UPLOAD_URL
551 cros_build_lib.Warning('unofficial builds upload to the staging server')
552 upload_url = STAGING_UPLOAD_URL
555 cros_build_lib.Info('uploading specified symbols to %s', upload_url)
557 if breakpad_dir is None:
558 breakpad_dir = os.path.join(
560 cros_generate_breakpad_symbols.FindBreakpadDir(board).lstrip('/'))
561 cros_build_lib.Info('uploading all symbols to %s from %s', upload_url,
563 sym_paths = [breakpad_dir]
565 # We use storage_query to ask the server about existing symbols. The
566 # storage_notify_proc process is used to post updates to the server. We
567 # cannot safely share the storage object between threads/processes, but
568 # we also want to minimize creating new ones as each object has to init
569 # new state (like server connections).
571 dedupe_limit = DEDUPE_LIMIT
572 dedupe_queue = multiprocessing.Queue()
573 storage_query = isolateserver.get_storage_api(constants.ISOLATESERVER,
577 dedupe_queue = storage_query = None
578 # Can't use parallel.BackgroundTaskRunner because that'll create multiple
579 # processes and we want only one the whole time (see comment above).
580 storage_notify_proc = multiprocessing.Process(
581 target=SymbolDeduplicatorNotify, args=(dedupe_namespace, dedupe_queue))
583 bg_errors = multiprocessing.Value('i')
584 watermark_errors = multiprocessing.Value('f')
585 failed_queue = multiprocessing.Queue()
586 uploader = functools.partial(
587 UploadSymbol, upload_url, file_limit=file_limit, sleep=sleep,
588 num_errors=bg_errors, watermark_errors=watermark_errors,
589 failed_queue=failed_queue, passed_queue=dedupe_queue)
591 start_time = datetime.datetime.now()
592 Counters = cros_build_lib.Collection(
593 'Counters', upload_limit=upload_limit, uploaded_count=0, deduped_count=0)
594 counters = Counters()
596 def _Upload(queue, counters, files):
601 for item in SymbolDeduplicator(storage_query, files):
604 if counters.upload_limit == 0:
608 counters.uploaded_count += 1
609 if counters.upload_limit is not None:
610 counters.upload_limit -= 1
612 counters.deduped_count += (len(files) - missing_count)
615 storage_notify_proc.start()
617 with osutils.TempDir(prefix='upload_symbols.') as tempdir:
618 # For the first run, we collect the symbols that failed. If the
619 # overall failure rate was low, we'll retry them on the second run.
620 for retry in (retry, False):
621 # We need to limit ourselves to one upload at a time to avoid the server
622 # kicking in DoS protection. See these bugs for more details:
623 # http://crbug.com/209442
624 # http://crbug.com/212496
625 with parallel.BackgroundTaskRunner(uploader, processes=1) as queue:
627 for sym_file in SymbolFinder(tempdir, sym_paths):
628 dedupe_list.append(sym_file)
629 dedupe_len = len(dedupe_list)
630 if dedupe_len < dedupe_limit:
631 if (counters.upload_limit is None or
632 dedupe_len < counters.upload_limit):
635 # We check the counter before _Upload so that we don't keep talking
636 # to the dedupe server. Otherwise, we end up sending one symbol at
637 # a time to it and that slows things down a lot.
638 if counters.upload_limit == 0:
641 _Upload(queue, counters, dedupe_list)
643 _Upload(queue, counters, dedupe_list)
645 # See if we need to retry, and if we haven't failed too many times yet.
646 if not retry or ErrorLimitHit(bg_errors, watermark_errors):
650 failed_queue.put(None)
652 sym_path = failed_queue.get()
655 sym_paths.append(sym_path)
658 cros_build_lib.Warning('retrying %i symbols', len(sym_paths))
659 if counters.upload_limit is not None:
660 counters.upload_limit += len(sym_paths)
661 # Decrement the error count in case we recover in the second pass.
662 assert bg_errors.value >= len(sym_paths), \
663 'more failed files than errors?'
664 bg_errors.value -= len(sym_paths)
666 # No failed symbols, so just return now.
669 # If the user has requested it, save all the symbol files that we failed to
670 # upload to a listing file. This should help with recovery efforts later.
671 failed_queue.put(None)
672 WriteQueueToFile(failed_list, failed_queue, breakpad_dir)
675 cros_build_lib.Info('finished uploading; joining background process')
677 dedupe_queue.put(None)
678 storage_notify_proc.join()
680 cros_build_lib.Info('uploaded %i symbols (%i were deduped) which took: %s',
681 counters.uploaded_count, counters.deduped_count,
682 datetime.datetime.now() - start_time)
684 return bg_errors.value
688 # TODO(build): Delete this assert.
689 assert isolateserver, 'Missing isolateserver import http://crbug.com/341152'
691 parser = commandline.ArgumentParser(description=__doc__)
693 parser.add_argument('sym_paths', type='path_or_uri', nargs='*', default=None,
694 help='symbol file or directory or URL or tarball')
695 parser.add_argument('--board', default=None,
696 help='board to build packages for')
697 parser.add_argument('--breakpad_root', type='path', default=None,
698 help='root directory for breakpad symbols')
699 parser.add_argument('--official_build', action='store_true', default=False,
700 help='point to official symbol server')
701 parser.add_argument('--regenerate', action='store_true', default=False,
702 help='regenerate all symbols')
703 parser.add_argument('--upload-limit', type=int, default=None,
704 help='only upload # number of symbols')
705 parser.add_argument('--strip_cfi', type=int,
706 default=CRASH_SERVER_FILE_LIMIT - (10 * 1024 * 1024),
707 help='strip CFI data for files above this size')
708 parser.add_argument('--failed-list', type='path',
709 help='where to save a list of failed symbols')
710 parser.add_argument('--dedupe', action='store_true', default=False,
711 help='use the swarming service to avoid re-uploading')
712 parser.add_argument('--testing', action='store_true', default=False,
713 help='run in testing mode')
714 parser.add_argument('--yes', action='store_true', default=False,
715 help='answer yes to all prompts')
717 opts = parser.parse_args(argv)
722 cros_build_lib.Die('--regenerate may not be used with specific files')
724 if opts.board is None:
725 cros_build_lib.Die('--board is required')
727 if opts.breakpad_root and opts.regenerate:
728 cros_build_lib.Die('--regenerate may not be used with --breakpad_root')
731 # TODO(build): Kill off --testing mode once unittests are up-to-snuff.
732 cros_build_lib.Info('running in testing mode')
733 # pylint: disable=W0601,W0603
734 global INITIAL_RETRY_DELAY, SymUpload, DEFAULT_SLEEP_DELAY
735 INITIAL_RETRY_DELAY = DEFAULT_SLEEP_DELAY = 0
736 SymUpload = TestingSymUpload
738 dedupe_namespace = None
740 if opts.official_build and not opts.testing:
741 dedupe_namespace = OFFICIAL_DEDUPE_NAMESPACE
743 dedupe_namespace = STAGING_DEDUPE_NAMESPACE
746 prolog = '\n'.join(textwrap.wrap(textwrap.dedent("""
747 Uploading symbols for an entire Chromium OS build is really only
748 necessary for release builds and in a few cases for developers
749 to debug problems. It will take considerable time to run. For
750 developer debugging purposes, consider instead passing specific
753 if not cros_build_lib.BooleanPrompt(
754 prompt='Are you sure you want to upload all build symbols',
755 default=False, prolog=prolog):
756 cros_build_lib.Die('better safe than sorry')
760 ret += cros_generate_breakpad_symbols.GenerateBreakpadSymbols(
761 opts.board, breakpad_dir=opts.breakpad_root)
763 ret += UploadSymbols(opts.board, official=opts.official_build,
764 breakpad_dir=opts.breakpad_root,
765 file_limit=opts.strip_cfi, sleep=DEFAULT_SLEEP_DELAY,
766 upload_limit=opts.upload_limit, sym_paths=opts.sym_paths,
767 failed_list=opts.failed_list,
768 dedupe_namespace=dedupe_namespace)
770 cros_build_lib.Error('encountered %i problem(s)', ret)
771 # Since exit(status) gets masked, clamp it to 1 so we don't inadvertently
772 # return 0 in case we are a multiple of the mask.
778 # We need this to run once per process. Do it at module import time as that
779 # will let us avoid doing it inline at function call time (see SymUpload) as
780 # that func might be called by the multiprocessing module which means we'll
781 # do the opener logic multiple times overall. Plus, if you're importing this
782 # module, it's a pretty good chance that you're going to need this.
783 poster.streaminghttp.register_openers()