tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 from buildman import builderthread
  21 from buildman import toolchain
  22 from patman import command
  23 from patman import gitutil
  24 from patman import terminal
  25 from patman.terminal import Print
  26
  27 # This indicates an new int or hex Kconfig property with no default
  28 # It hangs the build since the 'conf' tool cannot proceed without valid input.
  29 #
  30 # We get a repeat sequence of something like this:
  31 # >>
  32 # Break things (BREAK_ME) [] (NEW)
  33 # Error in reading or end of file.
  34 # <<
  35 # which indicates that BREAK_ME has an empty default
  36 RE_NO_DEFAULT = re.compile(b'\((\w+)\) \[] \(NEW\)')
  37
  38 """
  39 Theory of Operation
  40
  41 Please see README for user documentation, and you should be familiar with
  42 that before trying to make sense of this.
  43
  44 Buildman works by keeping the machine as busy as possible, building different
  45 commits for different boards on multiple CPUs at once.
  46
  47 The source repo (self.git_dir) contains all the commits to be built. Each
  48 thread works on a single board at a time. It checks out the first commit,
  49 configures it for that board, then builds it. Then it checks out the next
  50 commit and builds it (typically without re-configuring). When it runs out
  51 of commits, it gets another job from the builder and starts again with that
  52 board.
  53
  54 Clearly the builder threads could work either way - they could check out a
  55 commit and then built it for all boards. Using separate directories for each
  56 commit/board pair they could leave their build product around afterwards
  57 also.
  58
  59 The intent behind building a single board for multiple commits, is to make
  60 use of incremental builds. Since each commit is built incrementally from
  61 the previous one, builds are faster. Reconfiguring for a different board
  62 removes all intermediate object files.
  63
  64 Many threads can be working at once, but each has its own working directory.
  65 When a thread finishes a build, it puts the output files into a result
  66 directory.
  67
  68 The base directory used by buildman is normally '../<branch>', i.e.
  69 a directory higher than the source repository and named after the branch
  70 being built.
  71
  72 Within the base directory, we have one subdirectory for each commit. Within
  73 that is one subdirectory for each board. Within that is the build output for
  74 that commit/board combination.
  75
  76 Buildman also create working directories for each thread, in a .bm-work/
  77 subdirectory in the base dir.
  78
  79 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  80 'seaboard', and say that us-net has two commits. We will have directories
  81 like this:
  82
  83 us-net/             base directory
  84     01_g4ed4ebc_net--Add-tftp-speed-/
  85         sandbox/
  86             u-boot.bin
  87         seaboard/
  88             u-boot.bin
  89     02_g4ed4ebc_net--Check-tftp-comp/
  90         sandbox/
  91             u-boot.bin
  92         seaboard/
  93             u-boot.bin
  94     .bm-work/
  95         00/         working directory for thread 0 (contains source checkout)
  96             build/  build output
  97         01/         working directory for thread 1
  98             build/  build output
  99         ...
 100 u-boot/             source directory
 101     .git/           repository
 102 """
 103
 104 """Holds information about a particular error line we are outputing
 105
 106    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
 107        'w-' = fixed warning
 108    boards: List of Board objects which have line in the error/warning output
 109    errline: The text of the error line
 110 """
 111 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 112
 113 # Possible build outcomes
 114 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 115
 116 # Translate a commit subject into a valid filename (and handle unicode)
 117 trans_valid_chars = str.maketrans('/: ', '---')
 118
 119 BASE_CONFIG_FILENAMES = [
 120     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 121 ]
 122
 123 EXTRA_CONFIG_FILENAMES = [
 124     '.config', '.config-spl', '.config-tpl',
 125     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 126     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 127 ]
 128
 129 class Config:
 130     """Holds information about configuration settings for a board."""
 131     def __init__(self, config_filename, target):
 132         self.target = target
 133         self.config = {}
 134         for fname in config_filename:
 135             self.config[fname] = {}
 136
 137     def Add(self, fname, key, value):
 138         self.config[fname][key] = value
 139
 140     def __hash__(self):
 141         val = 0
 142         for fname in self.config:
 143             for key, value in self.config[fname].items():
 144                 print(key, value)
 145                 val = val ^ hash(key) & hash(value)
 146         return val
 147
 148 class Environment:
 149     """Holds information about environment variables for a board."""
 150     def __init__(self, target):
 151         self.target = target
 152         self.environment = {}
 153
 154     def Add(self, key, value):
 155         self.environment[key] = value
 156
 157 class Builder:
 158     """Class for building U-Boot for a particular commit.
 159
 160     Public members: (many should ->private)
 161         already_done: Number of builds already completed
 162         base_dir: Base directory to use for builder
 163         checkout: True to check out source, False to skip that step.
 164             This is used for testing.
 165         col: terminal.Color() object
 166         count: Number of commits to build
 167         do_make: Method to call to invoke Make
 168         fail: Number of builds that failed due to error
 169         force_build: Force building even if a build already exists
 170         force_config_on_failure: If a commit fails for a board, disable
 171             incremental building for the next commit we build for that
 172             board, so that we will see all warnings/errors again.
 173         force_build_failures: If a previously-built build (i.e. built on
 174             a previous run of buildman) is marked as failed, rebuild it.
 175         git_dir: Git directory containing source repository
 176         num_jobs: Number of jobs to run at once (passed to make as -j)
 177         num_threads: Number of builder threads to run
 178         out_queue: Queue of results to process
 179         re_make_err: Compiled regular expression for ignore_lines
 180         queue: Queue of jobs to run
 181         threads: List of active threads
 182         toolchains: Toolchains object to use for building
 183         upto: Current commit number we are building (0.count-1)
 184         warned: Number of builds that produced at least one warning
 185         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 186             incremental building, where buildman reconfigures on the first
 187             commit for a baord, and then just does an incremental build for
 188             the following commits. In fact buildman will reconfigure and
 189             retry for any failing commits, so generally the only effect of
 190             this option is to slow things down.
 191         in_tree: Build U-Boot in-tree instead of specifying an output
 192             directory separate from the source code. This option is really
 193             only useful for testing in-tree builds.
 194         work_in_output: Use the output directory as the work directory and
 195             don't write to a separate output directory.
 196         thread_exceptions: List of exceptions raised by thread jobs
 197
 198     Private members:
 199         _base_board_dict: Last-summarised Dict of boards
 200         _base_err_lines: Last-summarised list of errors
 201         _base_warn_lines: Last-summarised list of warnings
 202         _build_period_us: Time taken for a single build (float object).
 203         _complete_delay: Expected delay until completion (timedelta)
 204         _next_delay_update: Next time we plan to display a progress update
 205                 (datatime)
 206         _show_unknown: Show unknown boards (those not built) in summary
 207         _start_time: Start time for the build
 208         _timestamps: List of timestamps for the completion of the last
 209             last _timestamp_count builds. Each is a datetime object.
 210         _timestamp_count: Number of timestamps to keep in our list.
 211         _working_dir: Base working directory containing all threads
 212         _single_builder: BuilderThread object for the singer builder, if
 213             threading is not being used
 214         _terminated: Thread was terminated due to an error
 215         _restarting_config: True if 'Restart config' is detected in output
 216     """
 217     class Outcome:
 218         """Records a build outcome for a single make invocation
 219
 220         Public Members:
 221             rc: Outcome value (OUTCOME_...)
 222             err_lines: List of error lines or [] if none
 223             sizes: Dictionary of image size information, keyed by filename
 224                 - Each value is itself a dictionary containing
 225                     values for 'text', 'data' and 'bss', being the integer
 226                     size in bytes of each section.
 227             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 228                     value is itself a dictionary:
 229                         key: function name
 230                         value: Size of function in bytes
 231             config: Dictionary keyed by filename - e.g. '.config'. Each
 232                     value is itself a dictionary:
 233                         key: config name
 234                         value: config value
 235             environment: Dictionary keyed by environment variable, Each
 236                      value is the value of environment variable.
 237         """
 238         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 239                      environment):
 240             self.rc = rc
 241             self.err_lines = err_lines
 242             self.sizes = sizes
 243             self.func_sizes = func_sizes
 244             self.config = config
 245             self.environment = environment
 246
 247     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 248                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 249                  no_subdirs=False, full_path=False, verbose_build=False,
 250                  mrproper=False, per_board_out_dir=False,
 251                  config_only=False, squash_config_y=False,
 252                  warnings_as_errors=False, work_in_output=False,
 253                  test_thread_exceptions=False, adjust_cfg=None):
 254         """Create a new Builder object
 255
 256         Args:
 257             toolchains: Toolchains object to use for building
 258             base_dir: Base directory to use for builder
 259             git_dir: Git directory containing source repository
 260             num_threads: Number of builder threads to run
 261             num_jobs: Number of jobs to run at once (passed to make as -j)
 262             gnu_make: the command name of GNU Make.
 263             checkout: True to check out source, False to skip that step.
 264                 This is used for testing.
 265             show_unknown: Show unknown boards (those not built) in summary
 266             step: 1 to process every commit, n to process every nth commit
 267             no_subdirs: Don't create subdirectories when building current
 268                 source for a single board
 269             full_path: Return the full path in CROSS_COMPILE and don't set
 270                 PATH
 271             verbose_build: Run build with V=1 and don't use 'make -s'
 272             mrproper: Always run 'make mrproper' when configuring
 273             per_board_out_dir: Build in a separate persistent directory per
 274                 board rather than a thread-specific directory
 275             config_only: Only configure each build, don't build it
 276             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 277             warnings_as_errors: Treat all compiler warnings as errors
 278             work_in_output: Use the output directory as the work directory and
 279                 don't write to a separate output directory.
 280             test_thread_exceptions: Uses for tests only, True to make the
 281                 threads raise an exception instead of reporting their result.
 282                 This simulates a failure in the code somewhere
 283             adjust_cfg_list (list of str): List of changes to make to .config
 284                 file before building. Each is one of (where C is the config
 285                 option with or without the CONFIG_ prefix)
 286
 287                     C to enable C
 288                     ~C to disable C
 289                     C=val to set the value of C (val must have quotes if C is
 290                         a string Kconfig
 291
 292         """
 293         self.toolchains = toolchains
 294         self.base_dir = base_dir
 295         if work_in_output:
 296             self._working_dir = base_dir
 297         else:
 298             self._working_dir = os.path.join(base_dir, '.bm-work')
 299         self.threads = []
 300         self.do_make = self.Make
 301         self.gnu_make = gnu_make
 302         self.checkout = checkout
 303         self.num_threads = num_threads
 304         self.num_jobs = num_jobs
 305         self.already_done = 0
 306         self.force_build = False
 307         self.git_dir = git_dir
 308         self._show_unknown = show_unknown
 309         self._timestamp_count = 10
 310         self._build_period_us = None
 311         self._complete_delay = None
 312         self._next_delay_update = datetime.now()
 313         self._start_time = datetime.now()
 314         self.force_config_on_failure = True
 315         self.force_build_failures = False
 316         self.force_reconfig = False
 317         self._step = step
 318         self.in_tree = False
 319         self._error_lines = 0
 320         self.no_subdirs = no_subdirs
 321         self.full_path = full_path
 322         self.verbose_build = verbose_build
 323         self.config_only = config_only
 324         self.squash_config_y = squash_config_y
 325         self.config_filenames = BASE_CONFIG_FILENAMES
 326         self.work_in_output = work_in_output
 327         self.adjust_cfg = adjust_cfg
 328
 329         if not self.squash_config_y:
 330             self.config_filenames += EXTRA_CONFIG_FILENAMES
 331         self._terminated = False
 332         self._restarting_config = False
 333
 334         self.warnings_as_errors = warnings_as_errors
 335         self.col = terminal.Color()
 336
 337         self._re_function = re.compile('(.*): In function.*')
 338         self._re_files = re.compile('In file included from.*')
 339         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 340         self._re_dtb_warning = re.compile('(.*): Warning .*')
 341         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 342         self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
 343                                                 re.MULTILINE | re.DOTALL)
 344
 345         self.thread_exceptions = []
 346         self.test_thread_exceptions = test_thread_exceptions
 347         if self.num_threads:
 348             self._single_builder = None
 349             self.queue = queue.Queue()
 350             self.out_queue = queue.Queue()
 351             for i in range(self.num_threads):
 352                 t = builderthread.BuilderThread(
 353                         self, i, mrproper, per_board_out_dir,
 354                         test_exception=test_thread_exceptions)
 355                 t.setDaemon(True)
 356                 t.start()
 357                 self.threads.append(t)
 358
 359             t = builderthread.ResultThread(self)
 360             t.setDaemon(True)
 361             t.start()
 362             self.threads.append(t)
 363         else:
 364             self._single_builder = builderthread.BuilderThread(
 365                 self, -1, mrproper, per_board_out_dir)
 366
 367         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 368         self.re_make_err = re.compile('|'.join(ignore_lines))
 369
 370         # Handle existing graceful with SIGINT / Ctrl-C
 371         signal.signal(signal.SIGINT, self.signal_handler)
 372
 373     def __del__(self):
 374         """Get rid of all threads created by the builder"""
 375         for t in self.threads:
 376             del t
 377
 378     def signal_handler(self, signal, frame):
 379         sys.exit(1)
 380
 381     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 382                           show_detail=False, show_bloat=False,
 383                           list_error_boards=False, show_config=False,
 384                           show_environment=False, filter_dtb_warnings=False,
 385                           filter_migration_warnings=False):
 386         """Setup display options for the builder.
 387
 388         Args:
 389             show_errors: True to show summarised error/warning info
 390             show_sizes: Show size deltas
 391             show_detail: Show size delta detail for each board if show_sizes
 392             show_bloat: Show detail for each function
 393             list_error_boards: Show the boards which caused each error/warning
 394             show_config: Show config deltas
 395             show_environment: Show environment deltas
 396             filter_dtb_warnings: Filter out any warnings from the device-tree
 397                 compiler
 398             filter_migration_warnings: Filter out any warnings about migrating
 399                 a board to driver model
 400         """
 401         self._show_errors = show_errors
 402         self._show_sizes = show_sizes
 403         self._show_detail = show_detail
 404         self._show_bloat = show_bloat
 405         self._list_error_boards = list_error_boards
 406         self._show_config = show_config
 407         self._show_environment = show_environment
 408         self._filter_dtb_warnings = filter_dtb_warnings
 409         self._filter_migration_warnings = filter_migration_warnings
 410
 411     def _AddTimestamp(self):
 412         """Add a new timestamp to the list and record the build period.
 413
 414         The build period is the length of time taken to perform a single
 415         build (one board, one commit).
 416         """
 417         now = datetime.now()
 418         self._timestamps.append(now)
 419         count = len(self._timestamps)
 420         delta = self._timestamps[-1] - self._timestamps[0]
 421         seconds = delta.total_seconds()
 422
 423         # If we have enough data, estimate build period (time taken for a
 424         # single build) and therefore completion time.
 425         if count > 1 and self._next_delay_update < now:
 426             self._next_delay_update = now + timedelta(seconds=2)
 427             if seconds > 0:
 428                 self._build_period = float(seconds) / count
 429                 todo = self.count - self.upto
 430                 self._complete_delay = timedelta(microseconds=
 431                         self._build_period * todo * 1000000)
 432                 # Round it
 433                 self._complete_delay -= timedelta(
 434                         microseconds=self._complete_delay.microseconds)
 435
 436         if seconds > 60:
 437             self._timestamps.popleft()
 438             count -= 1
 439
 440     def SelectCommit(self, commit, checkout=True):
 441         """Checkout the selected commit for this build
 442         """
 443         self.commit = commit
 444         if checkout and self.checkout:
 445             gitutil.Checkout(commit.hash)
 446
 447     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 448         """Run make
 449
 450         Args:
 451             commit: Commit object that is being built
 452             brd: Board object that is being built
 453             stage: Stage that we are at (mrproper, config, build)
 454             cwd: Directory where make should be run
 455             args: Arguments to pass to make
 456             kwargs: Arguments to pass to command.RunPipe()
 457         """
 458
 459         def check_output(stream, data):
 460             if b'Restart config' in data:
 461                 self._restarting_config = True
 462
 463             # If we see 'Restart config' following by multiple errors
 464             if self._restarting_config:
 465                 m = RE_NO_DEFAULT.findall(data)
 466
 467                 # Number of occurences of each Kconfig item
 468                 multiple = [m.count(val) for val in set(m)]
 469
 470                 # If any of them occur more than once, we have a loop
 471                 if [val for val in multiple if val > 1]:
 472                     self._terminated = True
 473                     return True
 474             return False
 475
 476         self._restarting_config = False
 477         self._terminated  = False
 478         cmd = [self.gnu_make] + list(args)
 479         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 480                 cwd=cwd, raise_on_error=False, infile='/dev/null',
 481                 output_func=check_output, **kwargs)
 482
 483         if self._terminated:
 484             # Try to be helpful
 485             result.stderr += '(** did you define an int/hex Kconfig with no default? **)'
 486
 487         if self.verbose_build:
 488             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 489             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 490         return result
 491
 492     def ProcessResult(self, result):
 493         """Process the result of a build, showing progress information
 494
 495         Args:
 496             result: A CommandResult object, which indicates the result for
 497                     a single build
 498         """
 499         col = terminal.Color()
 500         if result:
 501             target = result.brd.target
 502
 503             self.upto += 1
 504             if result.return_code != 0:
 505                 self.fail += 1
 506             elif result.stderr:
 507                 self.warned += 1
 508             if result.already_done:
 509                 self.already_done += 1
 510             if self._verbose:
 511                 terminal.PrintClear()
 512                 boards_selected = {target : result.brd}
 513                 self.ResetResultSummary(boards_selected)
 514                 self.ProduceResultSummary(result.commit_upto, self.commits,
 515                                           boards_selected)
 516         else:
 517             target = '(starting)'
 518
 519         # Display separate counts for ok, warned and fail
 520         ok = self.upto - self.warned - self.fail
 521         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 522         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 523         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 524
 525         line += ' /%-5d  ' % self.count
 526         remaining = self.count - self.upto
 527         if remaining:
 528             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 529         else:
 530             line += ' ' * 8
 531
 532         # Add our current completion time estimate
 533         self._AddTimestamp()
 534         if self._complete_delay:
 535             line += '%s  : ' % self._complete_delay
 536
 537         line += target
 538         terminal.PrintClear()
 539         Print(line, newline=False, limit_to_line=True)
 540
 541     def _GetOutputDir(self, commit_upto):
 542         """Get the name of the output directory for a commit number
 543
 544         The output directory is typically .../<branch>/<commit>.
 545
 546         Args:
 547             commit_upto: Commit number to use (0..self.count-1)
 548         """
 549         if self.work_in_output:
 550             return self._working_dir
 551
 552         commit_dir = None
 553         if self.commits:
 554             commit = self.commits[commit_upto]
 555             subject = commit.subject.translate(trans_valid_chars)
 556             # See _GetOutputSpaceRemovals() which parses this name
 557             commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
 558                     commit.hash, subject[:20]))
 559         elif not self.no_subdirs:
 560             commit_dir = 'current'
 561         if not commit_dir:
 562             return self.base_dir
 563         return os.path.join(self.base_dir, commit_dir)
 564
 565     def GetBuildDir(self, commit_upto, target):
 566         """Get the name of the build directory for a commit number
 567
 568         The build directory is typically .../<branch>/<commit>/<target>.
 569
 570         Args:
 571             commit_upto: Commit number to use (0..self.count-1)
 572             target: Target name
 573         """
 574         output_dir = self._GetOutputDir(commit_upto)
 575         if self.work_in_output:
 576             return output_dir
 577         return os.path.join(output_dir, target)
 578
 579     def GetDoneFile(self, commit_upto, target):
 580         """Get the name of the done file for a commit number
 581
 582         Args:
 583             commit_upto: Commit number to use (0..self.count-1)
 584             target: Target name
 585         """
 586         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 587
 588     def GetSizesFile(self, commit_upto, target):
 589         """Get the name of the sizes file for a commit number
 590
 591         Args:
 592             commit_upto: Commit number to use (0..self.count-1)
 593             target: Target name
 594         """
 595         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 596
 597     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 598         """Get the name of the funcsizes file for a commit number and ELF file
 599
 600         Args:
 601             commit_upto: Commit number to use (0..self.count-1)
 602             target: Target name
 603             elf_fname: Filename of elf image
 604         """
 605         return os.path.join(self.GetBuildDir(commit_upto, target),
 606                             '%s.sizes' % elf_fname.replace('/', '-'))
 607
 608     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 609         """Get the name of the objdump file for a commit number and ELF file
 610
 611         Args:
 612             commit_upto: Commit number to use (0..self.count-1)
 613             target: Target name
 614             elf_fname: Filename of elf image
 615         """
 616         return os.path.join(self.GetBuildDir(commit_upto, target),
 617                             '%s.objdump' % elf_fname.replace('/', '-'))
 618
 619     def GetErrFile(self, commit_upto, target):
 620         """Get the name of the err file for a commit number
 621
 622         Args:
 623             commit_upto: Commit number to use (0..self.count-1)
 624             target: Target name
 625         """
 626         output_dir = self.GetBuildDir(commit_upto, target)
 627         return os.path.join(output_dir, 'err')
 628
 629     def FilterErrors(self, lines):
 630         """Filter out errors in which we have no interest
 631
 632         We should probably use map().
 633
 634         Args:
 635             lines: List of error lines, each a string
 636         Returns:
 637             New list with only interesting lines included
 638         """
 639         out_lines = []
 640         if self._filter_migration_warnings:
 641             text = '\n'.join(lines)
 642             text = self._re_migration_warning.sub('', text)
 643             lines = text.splitlines()
 644         for line in lines:
 645             if self.re_make_err.search(line):
 646                 continue
 647             if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
 648                 continue
 649             out_lines.append(line)
 650         return out_lines
 651
 652     def ReadFuncSizes(self, fname, fd):
 653         """Read function sizes from the output of 'nm'
 654
 655         Args:
 656             fd: File containing data to read
 657             fname: Filename we are reading from (just for errors)
 658
 659         Returns:
 660             Dictionary containing size of each function in bytes, indexed by
 661             function name.
 662         """
 663         sym = {}
 664         for line in fd.readlines():
 665             try:
 666                 if line.strip():
 667                     size, type, name = line[:-1].split()
 668             except:
 669                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 670                 continue
 671             if type in 'tTdDbB':
 672                 # function names begin with '.' on 64-bit powerpc
 673                 if '.' in name[1:]:
 674                     name = 'static.' + name.split('.')[0]
 675                 sym[name] = sym.get(name, 0) + int(size, 16)
 676         return sym
 677
 678     def _ProcessConfig(self, fname):
 679         """Read in a .config, autoconf.mk or autoconf.h file
 680
 681         This function handles all config file types. It ignores comments and
 682         any #defines which don't start with CONFIG_.
 683
 684         Args:
 685             fname: Filename to read
 686
 687         Returns:
 688             Dictionary:
 689                 key: Config name (e.g. CONFIG_DM)
 690                 value: Config value (e.g. 1)
 691         """
 692         config = {}
 693         if os.path.exists(fname):
 694             with open(fname) as fd:
 695                 for line in fd:
 696                     line = line.strip()
 697                     if line.startswith('#define'):
 698                         values = line[8:].split(' ', 1)
 699                         if len(values) > 1:
 700                             key, value = values
 701                         else:
 702                             key = values[0]
 703                             value = '1' if self.squash_config_y else ''
 704                         if not key.startswith('CONFIG_'):
 705                             continue
 706                     elif not line or line[0] in ['#', '*', '/']:
 707                         continue
 708                     else:
 709                         key, value = line.split('=', 1)
 710                     if self.squash_config_y and value == 'y':
 711                         value = '1'
 712                     config[key] = value
 713         return config
 714
 715     def _ProcessEnvironment(self, fname):
 716         """Read in a uboot.env file
 717
 718         This function reads in environment variables from a file.
 719
 720         Args:
 721             fname: Filename to read
 722
 723         Returns:
 724             Dictionary:
 725                 key: environment variable (e.g. bootlimit)
 726                 value: value of environment variable (e.g. 1)
 727         """
 728         environment = {}
 729         if os.path.exists(fname):
 730             with open(fname) as fd:
 731                 for line in fd.read().split('\0'):
 732                     try:
 733                         key, value = line.split('=', 1)
 734                         environment[key] = value
 735                     except ValueError:
 736                         # ignore lines we can't parse
 737                         pass
 738         return environment
 739
 740     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 741                         read_config, read_environment):
 742         """Work out the outcome of a build.
 743
 744         Args:
 745             commit_upto: Commit number to check (0..n-1)
 746             target: Target board to check
 747             read_func_sizes: True to read function size information
 748             read_config: True to read .config and autoconf.h files
 749             read_environment: True to read uboot.env files
 750
 751         Returns:
 752             Outcome object
 753         """
 754         done_file = self.GetDoneFile(commit_upto, target)
 755         sizes_file = self.GetSizesFile(commit_upto, target)
 756         sizes = {}
 757         func_sizes = {}
 758         config = {}
 759         environment = {}
 760         if os.path.exists(done_file):
 761             with open(done_file, 'r') as fd:
 762                 try:
 763                     return_code = int(fd.readline())
 764                 except ValueError:
 765                     # The file may be empty due to running out of disk space.
 766                     # Try a rebuild
 767                     return_code = 1
 768                 err_lines = []
 769                 err_file = self.GetErrFile(commit_upto, target)
 770                 if os.path.exists(err_file):
 771                     with open(err_file, 'r') as fd:
 772                         err_lines = self.FilterErrors(fd.readlines())
 773
 774                 # Decide whether the build was ok, failed or created warnings
 775                 if return_code:
 776                     rc = OUTCOME_ERROR
 777                 elif len(err_lines):
 778                     rc = OUTCOME_WARNING
 779                 else:
 780                     rc = OUTCOME_OK
 781
 782                 # Convert size information to our simple format
 783                 if os.path.exists(sizes_file):
 784                     with open(sizes_file, 'r') as fd:
 785                         for line in fd.readlines():
 786                             values = line.split()
 787                             rodata = 0
 788                             if len(values) > 6:
 789                                 rodata = int(values[6], 16)
 790                             size_dict = {
 791                                 'all' : int(values[0]) + int(values[1]) +
 792                                         int(values[2]),
 793                                 'text' : int(values[0]) - rodata,
 794                                 'data' : int(values[1]),
 795                                 'bss' : int(values[2]),
 796                                 'rodata' : rodata,
 797                             }
 798                             sizes[values[5]] = size_dict
 799
 800             if read_func_sizes:
 801                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 802                 for fname in glob.glob(pattern):
 803                     with open(fname, 'r') as fd:
 804                         dict_name = os.path.basename(fname).replace('.sizes',
 805                                                                     '')
 806                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 807
 808             if read_config:
 809                 output_dir = self.GetBuildDir(commit_upto, target)
 810                 for name in self.config_filenames:
 811                     fname = os.path.join(output_dir, name)
 812                     config[name] = self._ProcessConfig(fname)
 813
 814             if read_environment:
 815                 output_dir = self.GetBuildDir(commit_upto, target)
 816                 fname = os.path.join(output_dir, 'uboot.env')
 817                 environment = self._ProcessEnvironment(fname)
 818
 819             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 820                                    environment)
 821
 822         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 823
 824     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 825                          read_config, read_environment):
 826         """Calculate a summary of the results of building a commit.
 827
 828         Args:
 829             board_selected: Dict containing boards to summarise
 830             commit_upto: Commit number to summarize (0..self.count-1)
 831             read_func_sizes: True to read function size information
 832             read_config: True to read .config and autoconf.h files
 833             read_environment: True to read uboot.env files
 834
 835         Returns:
 836             Tuple:
 837                 Dict containing boards which passed building this commit.
 838                     keyed by board.target
 839                 List containing a summary of error lines
 840                 Dict keyed by error line, containing a list of the Board
 841                     objects with that error
 842                 List containing a summary of warning lines
 843                 Dict keyed by error line, containing a list of the Board
 844                     objects with that warning
 845                 Dictionary keyed by board.target. Each value is a dictionary:
 846                     key: filename - e.g. '.config'
 847                     value is itself a dictionary:
 848                         key: config name
 849                         value: config value
 850                 Dictionary keyed by board.target. Each value is a dictionary:
 851                     key: environment variable
 852                     value: value of environment variable
 853         """
 854         def AddLine(lines_summary, lines_boards, line, board):
 855             line = line.rstrip()
 856             if line in lines_boards:
 857                 lines_boards[line].append(board)
 858             else:
 859                 lines_boards[line] = [board]
 860                 lines_summary.append(line)
 861
 862         board_dict = {}
 863         err_lines_summary = []
 864         err_lines_boards = {}
 865         warn_lines_summary = []
 866         warn_lines_boards = {}
 867         config = {}
 868         environment = {}
 869
 870         for board in boards_selected.values():
 871             outcome = self.GetBuildOutcome(commit_upto, board.target,
 872                                            read_func_sizes, read_config,
 873                                            read_environment)
 874             board_dict[board.target] = outcome
 875             last_func = None
 876             last_was_warning = False
 877             for line in outcome.err_lines:
 878                 if line:
 879                     if (self._re_function.match(line) or
 880                             self._re_files.match(line)):
 881                         last_func = line
 882                     else:
 883                         is_warning = (self._re_warning.match(line) or
 884                                       self._re_dtb_warning.match(line))
 885                         is_note = self._re_note.match(line)
 886                         if is_warning or (last_was_warning and is_note):
 887                             if last_func:
 888                                 AddLine(warn_lines_summary, warn_lines_boards,
 889                                         last_func, board)
 890                             AddLine(warn_lines_summary, warn_lines_boards,
 891                                     line, board)
 892                         else:
 893                             if last_func:
 894                                 AddLine(err_lines_summary, err_lines_boards,
 895                                         last_func, board)
 896                             AddLine(err_lines_summary, err_lines_boards,
 897                                     line, board)
 898                         last_was_warning = is_warning
 899                         last_func = None
 900             tconfig = Config(self.config_filenames, board.target)
 901             for fname in self.config_filenames:
 902                 if outcome.config:
 903                     for key, value in outcome.config[fname].items():
 904                         tconfig.Add(fname, key, value)
 905             config[board.target] = tconfig
 906
 907             tenvironment = Environment(board.target)
 908             if outcome.environment:
 909                 for key, value in outcome.environment.items():
 910                     tenvironment.Add(key, value)
 911             environment[board.target] = tenvironment
 912
 913         return (board_dict, err_lines_summary, err_lines_boards,
 914                 warn_lines_summary, warn_lines_boards, config, environment)
 915
 916     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 917         """Add an output to our list of outcomes for each architecture
 918
 919         This simple function adds failing boards (changes) to the
 920         relevant architecture string, so we can print the results out
 921         sorted by architecture.
 922
 923         Args:
 924              board_dict: Dict containing all boards
 925              arch_list: Dict keyed by arch name. Value is a string containing
 926                     a list of board names which failed for that arch.
 927              changes: List of boards to add to arch_list
 928              color: terminal.Colour object
 929         """
 930         done_arch = {}
 931         for target in changes:
 932             if target in board_dict:
 933                 arch = board_dict[target].arch
 934             else:
 935                 arch = 'unknown'
 936             str = self.col.Color(color, ' ' + target)
 937             if not arch in done_arch:
 938                 str = ' %s  %s' % (self.col.Color(color, char), str)
 939                 done_arch[arch] = True
 940             if not arch in arch_list:
 941                 arch_list[arch] = str
 942             else:
 943                 arch_list[arch] += str
 944
 945
 946     def ColourNum(self, num):
 947         color = self.col.RED if num > 0 else self.col.GREEN
 948         if num == 0:
 949             return '0'
 950         return self.col.Color(color, str(num))
 951
 952     def ResetResultSummary(self, board_selected):
 953         """Reset the results summary ready for use.
 954
 955         Set up the base board list to be all those selected, and set the
 956         error lines to empty.
 957
 958         Following this, calls to PrintResultSummary() will use this
 959         information to work out what has changed.
 960
 961         Args:
 962             board_selected: Dict containing boards to summarise, keyed by
 963                 board.target
 964         """
 965         self._base_board_dict = {}
 966         for board in board_selected:
 967             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 968                                                            {})
 969         self._base_err_lines = []
 970         self._base_warn_lines = []
 971         self._base_err_line_boards = {}
 972         self._base_warn_line_boards = {}
 973         self._base_config = None
 974         self._base_environment = None
 975
 976     def PrintFuncSizeDetail(self, fname, old, new):
 977         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 978         delta, common = [], {}
 979
 980         for a in old:
 981             if a in new:
 982                 common[a] = 1
 983
 984         for name in old:
 985             if name not in common:
 986                 remove += 1
 987                 down += old[name]
 988                 delta.append([-old[name], name])
 989
 990         for name in new:
 991             if name not in common:
 992                 add += 1
 993                 up += new[name]
 994                 delta.append([new[name], name])
 995
 996         for name in common:
 997                 diff = new.get(name, 0) - old.get(name, 0)
 998                 if diff > 0:
 999                     grow, up = grow + 1, up + diff
1000                 elif diff < 0:
1001                     shrink, down = shrink + 1, down - diff
1002                 delta.append([diff, name])
1003
1004         delta.sort()
1005         delta.reverse()
1006
1007         args = [add, -remove, grow, -shrink, up, -down, up - down]
1008         if max(args) == 0 and min(args) == 0:
1009             return
1010         args = [self.ColourNum(x) for x in args]
1011         indent = ' ' * 15
1012         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
1013               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
1014         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
1015                                          'delta'))
1016         for diff, name in delta:
1017             if diff:
1018                 color = self.col.RED if diff > 0 else self.col.GREEN
1019                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
1020                         old.get(name, '-'), new.get(name,'-'), diff)
1021                 Print(msg, colour=color)
1022
1023
1024     def PrintSizeDetail(self, target_list, show_bloat):
1025         """Show details size information for each board
1026
1027         Args:
1028             target_list: List of targets, each a dict containing:
1029                     'target': Target name
1030                     'total_diff': Total difference in bytes across all areas
1031                     <part_name>: Difference for that part
1032             show_bloat: Show detail for each function
1033         """
1034         targets_by_diff = sorted(target_list, reverse=True,
1035         key=lambda x: x['_total_diff'])
1036         for result in targets_by_diff:
1037             printed_target = False
1038             for name in sorted(result):
1039                 diff = result[name]
1040                 if name.startswith('_'):
1041                     continue
1042                 if diff != 0:
1043                     color = self.col.RED if diff > 0 else self.col.GREEN
1044                 msg = ' %s %+d' % (name, diff)
1045                 if not printed_target:
1046                     Print('%10s  %-15s:' % ('', result['_target']),
1047                           newline=False)
1048                     printed_target = True
1049                 Print(msg, colour=color, newline=False)
1050             if printed_target:
1051                 Print()
1052                 if show_bloat:
1053                     target = result['_target']
1054                     outcome = result['_outcome']
1055                     base_outcome = self._base_board_dict[target]
1056                     for fname in outcome.func_sizes:
1057                         self.PrintFuncSizeDetail(fname,
1058                                                  base_outcome.func_sizes[fname],
1059                                                  outcome.func_sizes[fname])
1060
1061
1062     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
1063                          show_bloat):
1064         """Print a summary of image sizes broken down by section.
1065
1066         The summary takes the form of one line per architecture. The
1067         line contains deltas for each of the sections (+ means the section
1068         got bigger, - means smaller). The numbers are the average number
1069         of bytes that a board in this section increased by.
1070
1071         For example:
1072            powerpc: (622 boards)   text -0.0
1073           arm: (285 boards)   text -0.0
1074           nds32: (3 boards)   text -8.0
1075
1076         Args:
1077             board_selected: Dict containing boards to summarise, keyed by
1078                 board.target
1079             board_dict: Dict containing boards for which we built this
1080                 commit, keyed by board.target. The value is an Outcome object.
1081             show_detail: Show size delta detail for each board
1082             show_bloat: Show detail for each function
1083         """
1084         arch_list = {}
1085         arch_count = {}
1086
1087         # Calculate changes in size for different image parts
1088         # The previous sizes are in Board.sizes, for each board
1089         for target in board_dict:
1090             if target not in board_selected:
1091                 continue
1092             base_sizes = self._base_board_dict[target].sizes
1093             outcome = board_dict[target]
1094             sizes = outcome.sizes
1095
1096             # Loop through the list of images, creating a dict of size
1097             # changes for each image/part. We end up with something like
1098             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1099             # which means that U-Boot data increased by 5 bytes and SPL
1100             # text decreased by 4.
1101             err = {'_target' : target}
1102             for image in sizes:
1103                 if image in base_sizes:
1104                     base_image = base_sizes[image]
1105                     # Loop through the text, data, bss parts
1106                     for part in sorted(sizes[image]):
1107                         diff = sizes[image][part] - base_image[part]
1108                         col = None
1109                         if diff:
1110                             if image == 'u-boot':
1111                                 name = part
1112                             else:
1113                                 name = image + ':' + part
1114                             err[name] = diff
1115             arch = board_selected[target].arch
1116             if not arch in arch_count:
1117                 arch_count[arch] = 1
1118             else:
1119                 arch_count[arch] += 1
1120             if not sizes:
1121                 pass    # Only add to our list when we have some stats
1122             elif not arch in arch_list:
1123                 arch_list[arch] = [err]
1124             else:
1125                 arch_list[arch].append(err)
1126
1127         # We now have a list of image size changes sorted by arch
1128         # Print out a summary of these
1129         for arch, target_list in arch_list.items():
1130             # Get total difference for each type
1131             totals = {}
1132             for result in target_list:
1133                 total = 0
1134                 for name, diff in result.items():
1135                     if name.startswith('_'):
1136                         continue
1137                     total += diff
1138                     if name in totals:
1139                         totals[name] += diff
1140                     else:
1141                         totals[name] = diff
1142                 result['_total_diff'] = total
1143                 result['_outcome'] = board_dict[result['_target']]
1144
1145             count = len(target_list)
1146             printed_arch = False
1147             for name in sorted(totals):
1148                 diff = totals[name]
1149                 if diff:
1150                     # Display the average difference in this name for this
1151                     # architecture
1152                     avg_diff = float(diff) / count
1153                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1154                     msg = ' %s %+1.1f' % (name, avg_diff)
1155                     if not printed_arch:
1156                         Print('%10s: (for %d/%d boards)' % (arch, count,
1157                               arch_count[arch]), newline=False)
1158                         printed_arch = True
1159                     Print(msg, colour=color, newline=False)
1160
1161             if printed_arch:
1162                 Print()
1163                 if show_detail:
1164                     self.PrintSizeDetail(target_list, show_bloat)
1165
1166
1167     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1168                            err_line_boards, warn_lines, warn_line_boards,
1169                            config, environment, show_sizes, show_detail,
1170                            show_bloat, show_config, show_environment):
1171         """Compare results with the base results and display delta.
1172
1173         Only boards mentioned in board_selected will be considered. This
1174         function is intended to be called repeatedly with the results of
1175         each commit. It therefore shows a 'diff' between what it saw in
1176         the last call and what it sees now.
1177
1178         Args:
1179             board_selected: Dict containing boards to summarise, keyed by
1180                 board.target
1181             board_dict: Dict containing boards for which we built this
1182                 commit, keyed by board.target. The value is an Outcome object.
1183             err_lines: A list of errors for this commit, or [] if there is
1184                 none, or we don't want to print errors
1185             err_line_boards: Dict keyed by error line, containing a list of
1186                 the Board objects with that error
1187             warn_lines: A list of warnings for this commit, or [] if there is
1188                 none, or we don't want to print errors
1189             warn_line_boards: Dict keyed by warning line, containing a list of
1190                 the Board objects with that warning
1191             config: Dictionary keyed by filename - e.g. '.config'. Each
1192                     value is itself a dictionary:
1193                         key: config name
1194                         value: config value
1195             environment: Dictionary keyed by environment variable, Each
1196                      value is the value of environment variable.
1197             show_sizes: Show image size deltas
1198             show_detail: Show size delta detail for each board if show_sizes
1199             show_bloat: Show detail for each function
1200             show_config: Show config changes
1201             show_environment: Show environment changes
1202         """
1203         def _BoardList(line, line_boards):
1204             """Helper function to get a line of boards containing a line
1205
1206             Args:
1207                 line: Error line to search for
1208                 line_boards: boards to search, each a Board
1209             Return:
1210                 List of boards with that error line, or [] if the user has not
1211                     requested such a list
1212             """
1213             boards = []
1214             board_set = set()
1215             if self._list_error_boards:
1216                 for board in line_boards[line]:
1217                     if not board in board_set:
1218                         boards.append(board)
1219                         board_set.add(board)
1220             return boards
1221
1222         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1223                             char):
1224             """Calculate the required output based on changes in errors
1225
1226             Args:
1227                 base_lines: List of errors/warnings for previous commit
1228                 base_line_boards: Dict keyed by error line, containing a list
1229                     of the Board objects with that error in the previous commit
1230                 lines: List of errors/warning for this commit, each a str
1231                 line_boards: Dict keyed by error line, containing a list
1232                     of the Board objects with that error in this commit
1233                 char: Character representing error ('') or warning ('w'). The
1234                     broken ('+') or fixed ('-') characters are added in this
1235                     function
1236
1237             Returns:
1238                 Tuple
1239                     List of ErrLine objects for 'better' lines
1240                     List of ErrLine objects for 'worse' lines
1241             """
1242             better_lines = []
1243             worse_lines = []
1244             for line in lines:
1245                 if line not in base_lines:
1246                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1247                                       line)
1248                     worse_lines.append(errline)
1249             for line in base_lines:
1250                 if line not in lines:
1251                     errline = ErrLine(char + '-',
1252                                       _BoardList(line, base_line_boards), line)
1253                     better_lines.append(errline)
1254             return better_lines, worse_lines
1255
1256         def _CalcConfig(delta, name, config):
1257             """Calculate configuration changes
1258
1259             Args:
1260                 delta: Type of the delta, e.g. '+'
1261                 name: name of the file which changed (e.g. .config)
1262                 config: configuration change dictionary
1263                     key: config name
1264                     value: config value
1265             Returns:
1266                 String containing the configuration changes which can be
1267                     printed
1268             """
1269             out = ''
1270             for key in sorted(config.keys()):
1271                 out += '%s=%s ' % (key, config[key])
1272             return '%s %s: %s' % (delta, name, out)
1273
1274         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1275             """Add changes in configuration to a list
1276
1277             Args:
1278                 lines: list to add to
1279                 name: config file name
1280                 config_plus: configurations added, dictionary
1281                     key: config name
1282                     value: config value
1283                 config_minus: configurations removed, dictionary
1284                     key: config name
1285                     value: config value
1286                 config_change: configurations changed, dictionary
1287                     key: config name
1288                     value: config value
1289             """
1290             if config_plus:
1291                 lines.append(_CalcConfig('+', name, config_plus))
1292             if config_minus:
1293                 lines.append(_CalcConfig('-', name, config_minus))
1294             if config_change:
1295                 lines.append(_CalcConfig('c', name, config_change))
1296
1297         def _OutputConfigInfo(lines):
1298             for line in lines:
1299                 if not line:
1300                     continue
1301                 if line[0] == '+':
1302                     col = self.col.GREEN
1303                 elif line[0] == '-':
1304                     col = self.col.RED
1305                 elif line[0] == 'c':
1306                     col = self.col.YELLOW
1307                 Print('   ' + line, newline=True, colour=col)
1308
1309         def _OutputErrLines(err_lines, colour):
1310             """Output the line of error/warning lines, if not empty
1311
1312             Also increments self._error_lines if err_lines not empty
1313
1314             Args:
1315                 err_lines: List of ErrLine objects, each an error or warning
1316                     line, possibly including a list of boards with that
1317                     error/warning
1318                 colour: Colour to use for output
1319             """
1320             if err_lines:
1321                 out_list = []
1322                 for line in err_lines:
1323                     boards = ''
1324                     names = [board.target for board in line.boards]
1325                     board_str = ' '.join(names) if names else ''
1326                     if board_str:
1327                         out = self.col.Color(colour, line.char + '(')
1328                         out += self.col.Color(self.col.MAGENTA, board_str,
1329                                               bright=False)
1330                         out += self.col.Color(colour, ') %s' % line.errline)
1331                     else:
1332                         out = self.col.Color(colour, line.char + line.errline)
1333                     out_list.append(out)
1334                 Print('\n'.join(out_list))
1335                 self._error_lines += 1
1336
1337
1338         ok_boards = []      # List of boards fixed since last commit
1339         warn_boards = []    # List of boards with warnings since last commit
1340         err_boards = []     # List of new broken boards since last commit
1341         new_boards = []     # List of boards that didn't exist last time
1342         unknown_boards = [] # List of boards that were not built
1343
1344         for target in board_dict:
1345             if target not in board_selected:
1346                 continue
1347
1348             # If the board was built last time, add its outcome to a list
1349             if target in self._base_board_dict:
1350                 base_outcome = self._base_board_dict[target].rc
1351                 outcome = board_dict[target]
1352                 if outcome.rc == OUTCOME_UNKNOWN:
1353                     unknown_boards.append(target)
1354                 elif outcome.rc < base_outcome:
1355                     if outcome.rc == OUTCOME_WARNING:
1356                         warn_boards.append(target)
1357                     else:
1358                         ok_boards.append(target)
1359                 elif outcome.rc > base_outcome:
1360                     if outcome.rc == OUTCOME_WARNING:
1361                         warn_boards.append(target)
1362                     else:
1363                         err_boards.append(target)
1364             else:
1365                 new_boards.append(target)
1366
1367         # Get a list of errors and warnings that have appeared, and disappeared
1368         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1369                 self._base_err_line_boards, err_lines, err_line_boards, '')
1370         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1371                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1372
1373         # Display results by arch
1374         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1375                 worse_err, better_err, worse_warn, better_warn)):
1376             arch_list = {}
1377             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1378                     self.col.GREEN)
1379             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1380                     self.col.YELLOW)
1381             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1382                     self.col.RED)
1383             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1384             if self._show_unknown:
1385                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1386                         self.col.MAGENTA)
1387             for arch, target_list in arch_list.items():
1388                 Print('%10s: %s' % (arch, target_list))
1389                 self._error_lines += 1
1390             _OutputErrLines(better_err, colour=self.col.GREEN)
1391             _OutputErrLines(worse_err, colour=self.col.RED)
1392             _OutputErrLines(better_warn, colour=self.col.CYAN)
1393             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1394
1395         if show_sizes:
1396             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1397                                   show_bloat)
1398
1399         if show_environment and self._base_environment:
1400             lines = []
1401
1402             for target in board_dict:
1403                 if target not in board_selected:
1404                     continue
1405
1406                 tbase = self._base_environment[target]
1407                 tenvironment = environment[target]
1408                 environment_plus = {}
1409                 environment_minus = {}
1410                 environment_change = {}
1411                 base = tbase.environment
1412                 for key, value in tenvironment.environment.items():
1413                     if key not in base:
1414                         environment_plus[key] = value
1415                 for key, value in base.items():
1416                     if key not in tenvironment.environment:
1417                         environment_minus[key] = value
1418                 for key, value in base.items():
1419                     new_value = tenvironment.environment.get(key)
1420                     if new_value and value != new_value:
1421                         desc = '%s -> %s' % (value, new_value)
1422                         environment_change[key] = desc
1423
1424                 _AddConfig(lines, target, environment_plus, environment_minus,
1425                            environment_change)
1426
1427             _OutputConfigInfo(lines)
1428
1429         if show_config and self._base_config:
1430             summary = {}
1431             arch_config_plus = {}
1432             arch_config_minus = {}
1433             arch_config_change = {}
1434             arch_list = []
1435
1436             for target in board_dict:
1437                 if target not in board_selected:
1438                     continue
1439                 arch = board_selected[target].arch
1440                 if arch not in arch_list:
1441                     arch_list.append(arch)
1442
1443             for arch in arch_list:
1444                 arch_config_plus[arch] = {}
1445                 arch_config_minus[arch] = {}
1446                 arch_config_change[arch] = {}
1447                 for name in self.config_filenames:
1448                     arch_config_plus[arch][name] = {}
1449                     arch_config_minus[arch][name] = {}
1450                     arch_config_change[arch][name] = {}
1451
1452             for target in board_dict:
1453                 if target not in board_selected:
1454                     continue
1455
1456                 arch = board_selected[target].arch
1457
1458                 all_config_plus = {}
1459                 all_config_minus = {}
1460                 all_config_change = {}
1461                 tbase = self._base_config[target]
1462                 tconfig = config[target]
1463                 lines = []
1464                 for name in self.config_filenames:
1465                     if not tconfig.config[name]:
1466                         continue
1467                     config_plus = {}
1468                     config_minus = {}
1469                     config_change = {}
1470                     base = tbase.config[name]
1471                     for key, value in tconfig.config[name].items():
1472                         if key not in base:
1473                             config_plus[key] = value
1474                             all_config_plus[key] = value
1475                     for key, value in base.items():
1476                         if key not in tconfig.config[name]:
1477                             config_minus[key] = value
1478                             all_config_minus[key] = value
1479                     for key, value in base.items():
1480                         new_value = tconfig.config.get(key)
1481                         if new_value and value != new_value:
1482                             desc = '%s -> %s' % (value, new_value)
1483                             config_change[key] = desc
1484                             all_config_change[key] = desc
1485
1486                     arch_config_plus[arch][name].update(config_plus)
1487                     arch_config_minus[arch][name].update(config_minus)
1488                     arch_config_change[arch][name].update(config_change)
1489
1490                     _AddConfig(lines, name, config_plus, config_minus,
1491                                config_change)
1492                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1493                            all_config_change)
1494                 summary[target] = '\n'.join(lines)
1495
1496             lines_by_target = {}
1497             for target, lines in summary.items():
1498                 if lines in lines_by_target:
1499                     lines_by_target[lines].append(target)
1500                 else:
1501                     lines_by_target[lines] = [target]
1502
1503             for arch in arch_list:
1504                 lines = []
1505                 all_plus = {}
1506                 all_minus = {}
1507                 all_change = {}
1508                 for name in self.config_filenames:
1509                     all_plus.update(arch_config_plus[arch][name])
1510                     all_minus.update(arch_config_minus[arch][name])
1511                     all_change.update(arch_config_change[arch][name])
1512                     _AddConfig(lines, name, arch_config_plus[arch][name],
1513                                arch_config_minus[arch][name],
1514                                arch_config_change[arch][name])
1515                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1516                 #arch_summary[target] = '\n'.join(lines)
1517                 if lines:
1518                     Print('%s:' % arch)
1519                     _OutputConfigInfo(lines)
1520
1521             for lines, targets in lines_by_target.items():
1522                 if not lines:
1523                     continue
1524                 Print('%s :' % ' '.join(sorted(targets)))
1525                 _OutputConfigInfo(lines.split('\n'))
1526
1527
1528         # Save our updated information for the next call to this function
1529         self._base_board_dict = board_dict
1530         self._base_err_lines = err_lines
1531         self._base_warn_lines = warn_lines
1532         self._base_err_line_boards = err_line_boards
1533         self._base_warn_line_boards = warn_line_boards
1534         self._base_config = config
1535         self._base_environment = environment
1536
1537         # Get a list of boards that did not get built, if needed
1538         not_built = []
1539         for board in board_selected:
1540             if not board in board_dict:
1541                 not_built.append(board)
1542         if not_built:
1543             Print("Boards not built (%d): %s" % (len(not_built),
1544                   ', '.join(not_built)))
1545
1546     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1547             (board_dict, err_lines, err_line_boards, warn_lines,
1548              warn_line_boards, config, environment) = self.GetResultSummary(
1549                     board_selected, commit_upto,
1550                     read_func_sizes=self._show_bloat,
1551                     read_config=self._show_config,
1552                     read_environment=self._show_environment)
1553             if commits:
1554                 msg = '%02d: %s' % (commit_upto + 1,
1555                         commits[commit_upto].subject)
1556                 Print(msg, colour=self.col.BLUE)
1557             self.PrintResultSummary(board_selected, board_dict,
1558                     err_lines if self._show_errors else [], err_line_boards,
1559                     warn_lines if self._show_errors else [], warn_line_boards,
1560                     config, environment, self._show_sizes, self._show_detail,
1561                     self._show_bloat, self._show_config, self._show_environment)
1562
1563     def ShowSummary(self, commits, board_selected):
1564         """Show a build summary for U-Boot for a given board list.
1565
1566         Reset the result summary, then repeatedly call GetResultSummary on
1567         each commit's results, then display the differences we see.
1568
1569         Args:
1570             commit: Commit objects to summarise
1571             board_selected: Dict containing boards to summarise
1572         """
1573         self.commit_count = len(commits) if commits else 1
1574         self.commits = commits
1575         self.ResetResultSummary(board_selected)
1576         self._error_lines = 0
1577
1578         for commit_upto in range(0, self.commit_count, self._step):
1579             self.ProduceResultSummary(commit_upto, commits, board_selected)
1580         if not self._error_lines:
1581             Print('(no errors to report)', colour=self.col.GREEN)
1582
1583
1584     def SetupBuild(self, board_selected, commits):
1585         """Set up ready to start a build.
1586
1587         Args:
1588             board_selected: Selected boards to build
1589             commits: Selected commits to build
1590         """
1591         # First work out how many commits we will build
1592         count = (self.commit_count + self._step - 1) // self._step
1593         self.count = len(board_selected) * count
1594         self.upto = self.warned = self.fail = 0
1595         self._timestamps = collections.deque()
1596
1597     def GetThreadDir(self, thread_num):
1598         """Get the directory path to the working dir for a thread.
1599
1600         Args:
1601             thread_num: Number of thread to check (-1 for main process, which
1602                 is treated as 0)
1603         """
1604         if self.work_in_output:
1605             return self._working_dir
1606         return os.path.join(self._working_dir, '%02d' % max(thread_num, 0))
1607
1608     def _PrepareThread(self, thread_num, setup_git):
1609         """Prepare the working directory for a thread.
1610
1611         This clones or fetches the repo into the thread's work directory.
1612         Optionally, it can create a linked working tree of the repo in the
1613         thread's work directory instead.
1614
1615         Args:
1616             thread_num: Thread number (0, 1, ...)
1617             setup_git:
1618                'clone' to set up a git clone
1619                'worktree' to set up a git worktree
1620         """
1621         thread_dir = self.GetThreadDir(thread_num)
1622         builderthread.Mkdir(thread_dir)
1623         git_dir = os.path.join(thread_dir, '.git')
1624
1625         # Create a worktree or a git repo clone for this thread if it
1626         # doesn't already exist
1627         if setup_git and self.git_dir:
1628             src_dir = os.path.abspath(self.git_dir)
1629             if os.path.isdir(git_dir):
1630                 # This is a clone of the src_dir repo, we can keep using
1631                 # it but need to fetch from src_dir.
1632                 Print('\rFetching repo for thread %d' % thread_num,
1633                       newline=False)
1634                 gitutil.Fetch(git_dir, thread_dir)
1635                 terminal.PrintClear()
1636             elif os.path.isfile(git_dir):
1637                 # This is a worktree of the src_dir repo, we don't need to
1638                 # create it again or update it in any way.
1639                 pass
1640             elif os.path.exists(git_dir):
1641                 # Don't know what could trigger this, but we probably
1642                 # can't create a git worktree/clone here.
1643                 raise ValueError('Git dir %s exists, but is not a file '
1644                                  'or a directory.' % git_dir)
1645             elif setup_git == 'worktree':
1646                 Print('\rChecking out worktree for thread %d' % thread_num,
1647                       newline=False)
1648                 gitutil.AddWorktree(src_dir, thread_dir)
1649                 terminal.PrintClear()
1650             elif setup_git == 'clone' or setup_git == True:
1651                 Print('\rCloning repo for thread %d' % thread_num,
1652                       newline=False)
1653                 gitutil.Clone(src_dir, thread_dir)
1654                 terminal.PrintClear()
1655             else:
1656                 raise ValueError("Can't setup git repo with %s." % setup_git)
1657
1658     def _PrepareWorkingSpace(self, max_threads, setup_git):
1659         """Prepare the working directory for use.
1660
1661         Set up the git repo for each thread. Creates a linked working tree
1662         if git-worktree is available, or clones the repo if it isn't.
1663
1664         Args:
1665             max_threads: Maximum number of threads we expect to need. If 0 then
1666                 1 is set up, since the main process still needs somewhere to
1667                 work
1668             setup_git: True to set up a git worktree or a git clone
1669         """
1670         builderthread.Mkdir(self._working_dir)
1671         if setup_git and self.git_dir:
1672             src_dir = os.path.abspath(self.git_dir)
1673             if gitutil.CheckWorktreeIsAvailable(src_dir):
1674                 setup_git = 'worktree'
1675                 # If we previously added a worktree but the directory for it
1676                 # got deleted, we need to prune its files from the repo so
1677                 # that we can check out another in its place.
1678                 gitutil.PruneWorktrees(src_dir)
1679             else:
1680                 setup_git = 'clone'
1681
1682         # Always do at least one thread
1683         for thread in range(max(max_threads, 1)):
1684             self._PrepareThread(thread, setup_git)
1685
1686     def _GetOutputSpaceRemovals(self):
1687         """Get the output directories ready to receive files.
1688
1689         Figure out what needs to be deleted in the output directory before it
1690         can be used. We only delete old buildman directories which have the
1691         expected name pattern. See _GetOutputDir().
1692
1693         Returns:
1694             List of full paths of directories to remove
1695         """
1696         if not self.commits:
1697             return
1698         dir_list = []
1699         for commit_upto in range(self.commit_count):
1700             dir_list.append(self._GetOutputDir(commit_upto))
1701
1702         to_remove = []
1703         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1704             if dirname not in dir_list:
1705                 leaf = dirname[len(self.base_dir) + 1:]
1706                 m =  re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
1707                 if m:
1708                     to_remove.append(dirname)
1709         return to_remove
1710
1711     def _PrepareOutputSpace(self):
1712         """Get the output directories ready to receive files.
1713
1714         We delete any output directories which look like ones we need to
1715         create. Having left over directories is confusing when the user wants
1716         to check the output manually.
1717         """
1718         to_remove = self._GetOutputSpaceRemovals()
1719         if to_remove:
1720             Print('Removing %d old build directories...' % len(to_remove),
1721                   newline=False)
1722             for dirname in to_remove:
1723                 shutil.rmtree(dirname)
1724             terminal.PrintClear()
1725
1726     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1727         """Build all commits for a list of boards
1728
1729         Args:
1730             commits: List of commits to be build, each a Commit object
1731             boards_selected: Dict of selected boards, key is target name,
1732                     value is Board object
1733             keep_outputs: True to save build output files
1734             verbose: Display build results as they are completed
1735         Returns:
1736             Tuple containing:
1737                 - number of boards that failed to build
1738                 - number of boards that issued warnings
1739                 - list of thread exceptions raised
1740         """
1741         self.commit_count = len(commits) if commits else 1
1742         self.commits = commits
1743         self._verbose = verbose
1744
1745         self.ResetResultSummary(board_selected)
1746         builderthread.Mkdir(self.base_dir, parents = True)
1747         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1748                 commits is not None)
1749         self._PrepareOutputSpace()
1750         Print('\rStarting build...', newline=False)
1751         self.SetupBuild(board_selected, commits)
1752         self.ProcessResult(None)
1753         self.thread_exceptions = []
1754         # Create jobs to build all commits for each board
1755         for brd in board_selected.values():
1756             job = builderthread.BuilderJob()
1757             job.board = brd
1758             job.commits = commits
1759             job.keep_outputs = keep_outputs
1760             job.work_in_output = self.work_in_output
1761             job.adjust_cfg = self.adjust_cfg
1762             job.step = self._step
1763             if self.num_threads:
1764                 self.queue.put(job)
1765             else:
1766                 results = self._single_builder.RunJob(job)
1767
1768         if self.num_threads:
1769             term = threading.Thread(target=self.queue.join)
1770             term.setDaemon(True)
1771             term.start()
1772             while term.is_alive():
1773                 term.join(100)
1774
1775             # Wait until we have processed all output
1776             self.out_queue.join()
1777         Print()
1778
1779         msg = 'Completed: %d total built' % self.count
1780         if self.already_done:
1781            msg += ' (%d previously' % self.already_done
1782            if self.already_done != self.count:
1783                msg += ', %d newly' % (self.count - self.already_done)
1784            msg += ')'
1785         duration = datetime.now() - self._start_time
1786         if duration > timedelta(microseconds=1000000):
1787             if duration.microseconds >= 500000:
1788                 duration = duration + timedelta(seconds=1)
1789             duration = duration - timedelta(microseconds=duration.microseconds)
1790             rate = float(self.count) / duration.total_seconds()
1791             msg += ', duration %s, rate %1.2f' % (duration, rate)
1792         Print(msg)
1793         if self.thread_exceptions:
1794             Print('Failed: %d thread exceptions' % len(self.thread_exceptions),
1795                   colour=self.col.RED)
1796
1797         return (self.fail, self.warned, self.thread_exceptions)