tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185
 186     Private members:
 187         _base_board_dict: Last-summarised Dict of boards
 188         _base_err_lines: Last-summarised list of errors
 189         _base_warn_lines: Last-summarised list of warnings
 190         _build_period_us: Time taken for a single build (float object).
 191         _complete_delay: Expected delay until completion (timedelta)
 192         _next_delay_update: Next time we plan to display a progress update
 193                 (datatime)
 194         _show_unknown: Show unknown boards (those not built) in summary
 195         _start_time: Start time for the build
 196         _timestamps: List of timestamps for the completion of the last
 197             last _timestamp_count builds. Each is a datetime object.
 198         _timestamp_count: Number of timestamps to keep in our list.
 199         _working_dir: Base working directory containing all threads
 200     """
 201     class Outcome:
 202         """Records a build outcome for a single make invocation
 203
 204         Public Members:
 205             rc: Outcome value (OUTCOME_...)
 206             err_lines: List of error lines or [] if none
 207             sizes: Dictionary of image size information, keyed by filename
 208                 - Each value is itself a dictionary containing
 209                     values for 'text', 'data' and 'bss', being the integer
 210                     size in bytes of each section.
 211             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 212                     value is itself a dictionary:
 213                         key: function name
 214                         value: Size of function in bytes
 215             config: Dictionary keyed by filename - e.g. '.config'. Each
 216                     value is itself a dictionary:
 217                         key: config name
 218                         value: config value
 219             environment: Dictionary keyed by environment variable, Each
 220                      value is the value of environment variable.
 221         """
 222         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 223                      environment):
 224             self.rc = rc
 225             self.err_lines = err_lines
 226             self.sizes = sizes
 227             self.func_sizes = func_sizes
 228             self.config = config
 229             self.environment = environment
 230
 231     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 232                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 233                  no_subdirs=False, full_path=False, verbose_build=False,
 234                  incremental=False, per_board_out_dir=False,
 235                  config_only=False, squash_config_y=False,
 236                  warnings_as_errors=False, work_in_output=False):
 237         """Create a new Builder object
 238
 239         Args:
 240             toolchains: Toolchains object to use for building
 241             base_dir: Base directory to use for builder
 242             git_dir: Git directory containing source repository
 243             num_threads: Number of builder threads to run
 244             num_jobs: Number of jobs to run at once (passed to make as -j)
 245             gnu_make: the command name of GNU Make.
 246             checkout: True to check out source, False to skip that step.
 247                 This is used for testing.
 248             show_unknown: Show unknown boards (those not built) in summary
 249             step: 1 to process every commit, n to process every nth commit
 250             no_subdirs: Don't create subdirectories when building current
 251                 source for a single board
 252             full_path: Return the full path in CROSS_COMPILE and don't set
 253                 PATH
 254             verbose_build: Run build with V=1 and don't use 'make -s'
 255             incremental: Always perform incremental builds; don't run make
 256                 mrproper when configuring
 257             per_board_out_dir: Build in a separate persistent directory per
 258                 board rather than a thread-specific directory
 259             config_only: Only configure each build, don't build it
 260             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 261             warnings_as_errors: Treat all compiler warnings as errors
 262             work_in_output: Use the output directory as the work directory and
 263                 don't write to a separate output directory.
 264         """
 265         self.toolchains = toolchains
 266         self.base_dir = base_dir
 267         if work_in_output:
 268             self._working_dir = base_dir
 269         else:
 270             self._working_dir = os.path.join(base_dir, '.bm-work')
 271         self.threads = []
 272         self.do_make = self.Make
 273         self.gnu_make = gnu_make
 274         self.checkout = checkout
 275         self.num_threads = num_threads
 276         self.num_jobs = num_jobs
 277         self.already_done = 0
 278         self.force_build = False
 279         self.git_dir = git_dir
 280         self._show_unknown = show_unknown
 281         self._timestamp_count = 10
 282         self._build_period_us = None
 283         self._complete_delay = None
 284         self._next_delay_update = datetime.now()
 285         self._start_time = datetime.now()
 286         self.force_config_on_failure = True
 287         self.force_build_failures = False
 288         self.force_reconfig = False
 289         self._step = step
 290         self.in_tree = False
 291         self._error_lines = 0
 292         self.no_subdirs = no_subdirs
 293         self.full_path = full_path
 294         self.verbose_build = verbose_build
 295         self.config_only = config_only
 296         self.squash_config_y = squash_config_y
 297         self.config_filenames = BASE_CONFIG_FILENAMES
 298         self.work_in_output = work_in_output
 299         if not self.squash_config_y:
 300             self.config_filenames += EXTRA_CONFIG_FILENAMES
 301
 302         self.warnings_as_errors = warnings_as_errors
 303         self.col = terminal.Color()
 304
 305         self._re_function = re.compile('(.*): In function.*')
 306         self._re_files = re.compile('In file included from.*')
 307         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 308         self._re_dtb_warning = re.compile('(.*): Warning .*')
 309         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 310
 311         self.queue = queue.Queue()
 312         self.out_queue = queue.Queue()
 313         for i in range(self.num_threads):
 314             t = builderthread.BuilderThread(self, i, incremental,
 315                     per_board_out_dir)
 316             t.setDaemon(True)
 317             t.start()
 318             self.threads.append(t)
 319
 320         t = builderthread.ResultThread(self)
 321         t.setDaemon(True)
 322         t.start()
 323         self.threads.append(t)
 324
 325         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 326         self.re_make_err = re.compile('|'.join(ignore_lines))
 327
 328         # Handle existing graceful with SIGINT / Ctrl-C
 329         signal.signal(signal.SIGINT, self.signal_handler)
 330
 331     def __del__(self):
 332         """Get rid of all threads created by the builder"""
 333         for t in self.threads:
 334             del t
 335
 336     def signal_handler(self, signal, frame):
 337         sys.exit(1)
 338
 339     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 340                           show_detail=False, show_bloat=False,
 341                           list_error_boards=False, show_config=False,
 342                           show_environment=False):
 343         """Setup display options for the builder.
 344
 345         show_errors: True to show summarised error/warning info
 346         show_sizes: Show size deltas
 347         show_detail: Show size delta detail for each board if show_sizes
 348         show_bloat: Show detail for each function
 349         list_error_boards: Show the boards which caused each error/warning
 350         show_config: Show config deltas
 351         show_environment: Show environment deltas
 352         """
 353         self._show_errors = show_errors
 354         self._show_sizes = show_sizes
 355         self._show_detail = show_detail
 356         self._show_bloat = show_bloat
 357         self._list_error_boards = list_error_boards
 358         self._show_config = show_config
 359         self._show_environment = show_environment
 360
 361     def _AddTimestamp(self):
 362         """Add a new timestamp to the list and record the build period.
 363
 364         The build period is the length of time taken to perform a single
 365         build (one board, one commit).
 366         """
 367         now = datetime.now()
 368         self._timestamps.append(now)
 369         count = len(self._timestamps)
 370         delta = self._timestamps[-1] - self._timestamps[0]
 371         seconds = delta.total_seconds()
 372
 373         # If we have enough data, estimate build period (time taken for a
 374         # single build) and therefore completion time.
 375         if count > 1 and self._next_delay_update < now:
 376             self._next_delay_update = now + timedelta(seconds=2)
 377             if seconds > 0:
 378                 self._build_period = float(seconds) / count
 379                 todo = self.count - self.upto
 380                 self._complete_delay = timedelta(microseconds=
 381                         self._build_period * todo * 1000000)
 382                 # Round it
 383                 self._complete_delay -= timedelta(
 384                         microseconds=self._complete_delay.microseconds)
 385
 386         if seconds > 60:
 387             self._timestamps.popleft()
 388             count -= 1
 389
 390     def SelectCommit(self, commit, checkout=True):
 391         """Checkout the selected commit for this build
 392         """
 393         self.commit = commit
 394         if checkout and self.checkout:
 395             gitutil.Checkout(commit.hash)
 396
 397     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 398         """Run make
 399
 400         Args:
 401             commit: Commit object that is being built
 402             brd: Board object that is being built
 403             stage: Stage that we are at (mrproper, config, build)
 404             cwd: Directory where make should be run
 405             args: Arguments to pass to make
 406             kwargs: Arguments to pass to command.RunPipe()
 407         """
 408         cmd = [self.gnu_make] + list(args)
 409         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 410                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 411         if self.verbose_build:
 412             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 413             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 414         return result
 415
 416     def ProcessResult(self, result):
 417         """Process the result of a build, showing progress information
 418
 419         Args:
 420             result: A CommandResult object, which indicates the result for
 421                     a single build
 422         """
 423         col = terminal.Color()
 424         if result:
 425             target = result.brd.target
 426
 427             self.upto += 1
 428             if result.return_code != 0:
 429                 self.fail += 1
 430             elif result.stderr:
 431                 self.warned += 1
 432             if result.already_done:
 433                 self.already_done += 1
 434             if self._verbose:
 435                 terminal.PrintClear()
 436                 boards_selected = {target : result.brd}
 437                 self.ResetResultSummary(boards_selected)
 438                 self.ProduceResultSummary(result.commit_upto, self.commits,
 439                                           boards_selected)
 440         else:
 441             target = '(starting)'
 442
 443         # Display separate counts for ok, warned and fail
 444         ok = self.upto - self.warned - self.fail
 445         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 446         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 447         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 448
 449         line += ' /%-5d  ' % self.count
 450         remaining = self.count - self.upto
 451         if remaining:
 452             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 453         else:
 454             line += ' ' * 8
 455
 456         # Add our current completion time estimate
 457         self._AddTimestamp()
 458         if self._complete_delay:
 459             line += '%s  : ' % self._complete_delay
 460
 461         line += target
 462         terminal.PrintClear()
 463         Print(line, newline=False, limit_to_line=True)
 464
 465     def _GetOutputDir(self, commit_upto):
 466         """Get the name of the output directory for a commit number
 467
 468         The output directory is typically .../<branch>/<commit>.
 469
 470         Args:
 471             commit_upto: Commit number to use (0..self.count-1)
 472         """
 473         commit_dir = None
 474         if self.commits:
 475             commit = self.commits[commit_upto]
 476             subject = commit.subject.translate(trans_valid_chars)
 477             # See _GetOutputSpaceRemovals() which parses this name
 478             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 479                     self.commit_count, commit.hash, subject[:20]))
 480         elif not self.no_subdirs:
 481             commit_dir = 'current'
 482         if not commit_dir:
 483             return self.base_dir
 484         return os.path.join(self.base_dir, commit_dir)
 485
 486     def GetBuildDir(self, commit_upto, target):
 487         """Get the name of the build directory for a commit number
 488
 489         The build directory is typically .../<branch>/<commit>/<target>.
 490
 491         Args:
 492             commit_upto: Commit number to use (0..self.count-1)
 493             target: Target name
 494         """
 495         output_dir = self._GetOutputDir(commit_upto)
 496         return os.path.join(output_dir, target)
 497
 498     def GetDoneFile(self, commit_upto, target):
 499         """Get the name of the done file for a commit number
 500
 501         Args:
 502             commit_upto: Commit number to use (0..self.count-1)
 503             target: Target name
 504         """
 505         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 506
 507     def GetSizesFile(self, commit_upto, target):
 508         """Get the name of the sizes file for a commit number
 509
 510         Args:
 511             commit_upto: Commit number to use (0..self.count-1)
 512             target: Target name
 513         """
 514         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 515
 516     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 517         """Get the name of the funcsizes file for a commit number and ELF file
 518
 519         Args:
 520             commit_upto: Commit number to use (0..self.count-1)
 521             target: Target name
 522             elf_fname: Filename of elf image
 523         """
 524         return os.path.join(self.GetBuildDir(commit_upto, target),
 525                             '%s.sizes' % elf_fname.replace('/', '-'))
 526
 527     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 528         """Get the name of the objdump file for a commit number and ELF file
 529
 530         Args:
 531             commit_upto: Commit number to use (0..self.count-1)
 532             target: Target name
 533             elf_fname: Filename of elf image
 534         """
 535         return os.path.join(self.GetBuildDir(commit_upto, target),
 536                             '%s.objdump' % elf_fname.replace('/', '-'))
 537
 538     def GetErrFile(self, commit_upto, target):
 539         """Get the name of the err file for a commit number
 540
 541         Args:
 542             commit_upto: Commit number to use (0..self.count-1)
 543             target: Target name
 544         """
 545         output_dir = self.GetBuildDir(commit_upto, target)
 546         return os.path.join(output_dir, 'err')
 547
 548     def FilterErrors(self, lines):
 549         """Filter out errors in which we have no interest
 550
 551         We should probably use map().
 552
 553         Args:
 554             lines: List of error lines, each a string
 555         Returns:
 556             New list with only interesting lines included
 557         """
 558         out_lines = []
 559         for line in lines:
 560             if not self.re_make_err.search(line):
 561                 out_lines.append(line)
 562         return out_lines
 563
 564     def ReadFuncSizes(self, fname, fd):
 565         """Read function sizes from the output of 'nm'
 566
 567         Args:
 568             fd: File containing data to read
 569             fname: Filename we are reading from (just for errors)
 570
 571         Returns:
 572             Dictionary containing size of each function in bytes, indexed by
 573             function name.
 574         """
 575         sym = {}
 576         for line in fd.readlines():
 577             try:
 578                 if line.strip():
 579                     size, type, name = line[:-1].split()
 580             except:
 581                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 582                 continue
 583             if type in 'tTdDbB':
 584                 # function names begin with '.' on 64-bit powerpc
 585                 if '.' in name[1:]:
 586                     name = 'static.' + name.split('.')[0]
 587                 sym[name] = sym.get(name, 0) + int(size, 16)
 588         return sym
 589
 590     def _ProcessConfig(self, fname):
 591         """Read in a .config, autoconf.mk or autoconf.h file
 592
 593         This function handles all config file types. It ignores comments and
 594         any #defines which don't start with CONFIG_.
 595
 596         Args:
 597             fname: Filename to read
 598
 599         Returns:
 600             Dictionary:
 601                 key: Config name (e.g. CONFIG_DM)
 602                 value: Config value (e.g. 1)
 603         """
 604         config = {}
 605         if os.path.exists(fname):
 606             with open(fname) as fd:
 607                 for line in fd:
 608                     line = line.strip()
 609                     if line.startswith('#define'):
 610                         values = line[8:].split(' ', 1)
 611                         if len(values) > 1:
 612                             key, value = values
 613                         else:
 614                             key = values[0]
 615                             value = '1' if self.squash_config_y else ''
 616                         if not key.startswith('CONFIG_'):
 617                             continue
 618                     elif not line or line[0] in ['#', '*', '/']:
 619                         continue
 620                     else:
 621                         key, value = line.split('=', 1)
 622                     if self.squash_config_y and value == 'y':
 623                         value = '1'
 624                     config[key] = value
 625         return config
 626
 627     def _ProcessEnvironment(self, fname):
 628         """Read in a uboot.env file
 629
 630         This function reads in environment variables from a file.
 631
 632         Args:
 633             fname: Filename to read
 634
 635         Returns:
 636             Dictionary:
 637                 key: environment variable (e.g. bootlimit)
 638                 value: value of environment variable (e.g. 1)
 639         """
 640         environment = {}
 641         if os.path.exists(fname):
 642             with open(fname) as fd:
 643                 for line in fd.read().split('\0'):
 644                     try:
 645                         key, value = line.split('=', 1)
 646                         environment[key] = value
 647                     except ValueError:
 648                         # ignore lines we can't parse
 649                         pass
 650         return environment
 651
 652     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 653                         read_config, read_environment):
 654         """Work out the outcome of a build.
 655
 656         Args:
 657             commit_upto: Commit number to check (0..n-1)
 658             target: Target board to check
 659             read_func_sizes: True to read function size information
 660             read_config: True to read .config and autoconf.h files
 661             read_environment: True to read uboot.env files
 662
 663         Returns:
 664             Outcome object
 665         """
 666         done_file = self.GetDoneFile(commit_upto, target)
 667         sizes_file = self.GetSizesFile(commit_upto, target)
 668         sizes = {}
 669         func_sizes = {}
 670         config = {}
 671         environment = {}
 672         if os.path.exists(done_file):
 673             with open(done_file, 'r') as fd:
 674                 try:
 675                     return_code = int(fd.readline())
 676                 except ValueError:
 677                     # The file may be empty due to running out of disk space.
 678                     # Try a rebuild
 679                     return_code = 1
 680                 err_lines = []
 681                 err_file = self.GetErrFile(commit_upto, target)
 682                 if os.path.exists(err_file):
 683                     with open(err_file, 'r') as fd:
 684                         err_lines = self.FilterErrors(fd.readlines())
 685
 686                 # Decide whether the build was ok, failed or created warnings
 687                 if return_code:
 688                     rc = OUTCOME_ERROR
 689                 elif len(err_lines):
 690                     rc = OUTCOME_WARNING
 691                 else:
 692                     rc = OUTCOME_OK
 693
 694                 # Convert size information to our simple format
 695                 if os.path.exists(sizes_file):
 696                     with open(sizes_file, 'r') as fd:
 697                         for line in fd.readlines():
 698                             values = line.split()
 699                             rodata = 0
 700                             if len(values) > 6:
 701                                 rodata = int(values[6], 16)
 702                             size_dict = {
 703                                 'all' : int(values[0]) + int(values[1]) +
 704                                         int(values[2]),
 705                                 'text' : int(values[0]) - rodata,
 706                                 'data' : int(values[1]),
 707                                 'bss' : int(values[2]),
 708                                 'rodata' : rodata,
 709                             }
 710                             sizes[values[5]] = size_dict
 711
 712             if read_func_sizes:
 713                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 714                 for fname in glob.glob(pattern):
 715                     with open(fname, 'r') as fd:
 716                         dict_name = os.path.basename(fname).replace('.sizes',
 717                                                                     '')
 718                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 719
 720             if read_config:
 721                 output_dir = self.GetBuildDir(commit_upto, target)
 722                 for name in self.config_filenames:
 723                     fname = os.path.join(output_dir, name)
 724                     config[name] = self._ProcessConfig(fname)
 725
 726             if read_environment:
 727                 output_dir = self.GetBuildDir(commit_upto, target)
 728                 fname = os.path.join(output_dir, 'uboot.env')
 729                 environment = self._ProcessEnvironment(fname)
 730
 731             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 732                                    environment)
 733
 734         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 735
 736     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 737                          read_config, read_environment):
 738         """Calculate a summary of the results of building a commit.
 739
 740         Args:
 741             board_selected: Dict containing boards to summarise
 742             commit_upto: Commit number to summarize (0..self.count-1)
 743             read_func_sizes: True to read function size information
 744             read_config: True to read .config and autoconf.h files
 745             read_environment: True to read uboot.env files
 746
 747         Returns:
 748             Tuple:
 749                 Dict containing boards which passed building this commit.
 750                     keyed by board.target
 751                 List containing a summary of error lines
 752                 Dict keyed by error line, containing a list of the Board
 753                     objects with that error
 754                 List containing a summary of warning lines
 755                 Dict keyed by error line, containing a list of the Board
 756                     objects with that warning
 757                 Dictionary keyed by board.target. Each value is a dictionary:
 758                     key: filename - e.g. '.config'
 759                     value is itself a dictionary:
 760                         key: config name
 761                         value: config value
 762                 Dictionary keyed by board.target. Each value is a dictionary:
 763                     key: environment variable
 764                     value: value of environment variable
 765         """
 766         def AddLine(lines_summary, lines_boards, line, board):
 767             line = line.rstrip()
 768             if line in lines_boards:
 769                 lines_boards[line].append(board)
 770             else:
 771                 lines_boards[line] = [board]
 772                 lines_summary.append(line)
 773
 774         board_dict = {}
 775         err_lines_summary = []
 776         err_lines_boards = {}
 777         warn_lines_summary = []
 778         warn_lines_boards = {}
 779         config = {}
 780         environment = {}
 781
 782         for board in boards_selected.values():
 783             outcome = self.GetBuildOutcome(commit_upto, board.target,
 784                                            read_func_sizes, read_config,
 785                                            read_environment)
 786             board_dict[board.target] = outcome
 787             last_func = None
 788             last_was_warning = False
 789             for line in outcome.err_lines:
 790                 if line:
 791                     if (self._re_function.match(line) or
 792                             self._re_files.match(line)):
 793                         last_func = line
 794                     else:
 795                         is_warning = (self._re_warning.match(line) or
 796                                       self._re_dtb_warning.match(line))
 797                         is_note = self._re_note.match(line)
 798                         if is_warning or (last_was_warning and is_note):
 799                             if last_func:
 800                                 AddLine(warn_lines_summary, warn_lines_boards,
 801                                         last_func, board)
 802                             AddLine(warn_lines_summary, warn_lines_boards,
 803                                     line, board)
 804                         else:
 805                             if last_func:
 806                                 AddLine(err_lines_summary, err_lines_boards,
 807                                         last_func, board)
 808                             AddLine(err_lines_summary, err_lines_boards,
 809                                     line, board)
 810                         last_was_warning = is_warning
 811                         last_func = None
 812             tconfig = Config(self.config_filenames, board.target)
 813             for fname in self.config_filenames:
 814                 if outcome.config:
 815                     for key, value in outcome.config[fname].items():
 816                         tconfig.Add(fname, key, value)
 817             config[board.target] = tconfig
 818
 819             tenvironment = Environment(board.target)
 820             if outcome.environment:
 821                 for key, value in outcome.environment.items():
 822                     tenvironment.Add(key, value)
 823             environment[board.target] = tenvironment
 824
 825         return (board_dict, err_lines_summary, err_lines_boards,
 826                 warn_lines_summary, warn_lines_boards, config, environment)
 827
 828     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 829         """Add an output to our list of outcomes for each architecture
 830
 831         This simple function adds failing boards (changes) to the
 832         relevant architecture string, so we can print the results out
 833         sorted by architecture.
 834
 835         Args:
 836              board_dict: Dict containing all boards
 837              arch_list: Dict keyed by arch name. Value is a string containing
 838                     a list of board names which failed for that arch.
 839              changes: List of boards to add to arch_list
 840              color: terminal.Colour object
 841         """
 842         done_arch = {}
 843         for target in changes:
 844             if target in board_dict:
 845                 arch = board_dict[target].arch
 846             else:
 847                 arch = 'unknown'
 848             str = self.col.Color(color, ' ' + target)
 849             if not arch in done_arch:
 850                 str = ' %s  %s' % (self.col.Color(color, char), str)
 851                 done_arch[arch] = True
 852             if not arch in arch_list:
 853                 arch_list[arch] = str
 854             else:
 855                 arch_list[arch] += str
 856
 857
 858     def ColourNum(self, num):
 859         color = self.col.RED if num > 0 else self.col.GREEN
 860         if num == 0:
 861             return '0'
 862         return self.col.Color(color, str(num))
 863
 864     def ResetResultSummary(self, board_selected):
 865         """Reset the results summary ready for use.
 866
 867         Set up the base board list to be all those selected, and set the
 868         error lines to empty.
 869
 870         Following this, calls to PrintResultSummary() will use this
 871         information to work out what has changed.
 872
 873         Args:
 874             board_selected: Dict containing boards to summarise, keyed by
 875                 board.target
 876         """
 877         self._base_board_dict = {}
 878         for board in board_selected:
 879             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 880                                                            {})
 881         self._base_err_lines = []
 882         self._base_warn_lines = []
 883         self._base_err_line_boards = {}
 884         self._base_warn_line_boards = {}
 885         self._base_config = None
 886         self._base_environment = None
 887
 888     def PrintFuncSizeDetail(self, fname, old, new):
 889         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 890         delta, common = [], {}
 891
 892         for a in old:
 893             if a in new:
 894                 common[a] = 1
 895
 896         for name in old:
 897             if name not in common:
 898                 remove += 1
 899                 down += old[name]
 900                 delta.append([-old[name], name])
 901
 902         for name in new:
 903             if name not in common:
 904                 add += 1
 905                 up += new[name]
 906                 delta.append([new[name], name])
 907
 908         for name in common:
 909                 diff = new.get(name, 0) - old.get(name, 0)
 910                 if diff > 0:
 911                     grow, up = grow + 1, up + diff
 912                 elif diff < 0:
 913                     shrink, down = shrink + 1, down - diff
 914                 delta.append([diff, name])
 915
 916         delta.sort()
 917         delta.reverse()
 918
 919         args = [add, -remove, grow, -shrink, up, -down, up - down]
 920         if max(args) == 0 and min(args) == 0:
 921             return
 922         args = [self.ColourNum(x) for x in args]
 923         indent = ' ' * 15
 924         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 925               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 926         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 927                                          'delta'))
 928         for diff, name in delta:
 929             if diff:
 930                 color = self.col.RED if diff > 0 else self.col.GREEN
 931                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 932                         old.get(name, '-'), new.get(name,'-'), diff)
 933                 Print(msg, colour=color)
 934
 935
 936     def PrintSizeDetail(self, target_list, show_bloat):
 937         """Show details size information for each board
 938
 939         Args:
 940             target_list: List of targets, each a dict containing:
 941                     'target': Target name
 942                     'total_diff': Total difference in bytes across all areas
 943                     <part_name>: Difference for that part
 944             show_bloat: Show detail for each function
 945         """
 946         targets_by_diff = sorted(target_list, reverse=True,
 947         key=lambda x: x['_total_diff'])
 948         for result in targets_by_diff:
 949             printed_target = False
 950             for name in sorted(result):
 951                 diff = result[name]
 952                 if name.startswith('_'):
 953                     continue
 954                 if diff != 0:
 955                     color = self.col.RED if diff > 0 else self.col.GREEN
 956                 msg = ' %s %+d' % (name, diff)
 957                 if not printed_target:
 958                     Print('%10s  %-15s:' % ('', result['_target']),
 959                           newline=False)
 960                     printed_target = True
 961                 Print(msg, colour=color, newline=False)
 962             if printed_target:
 963                 Print()
 964                 if show_bloat:
 965                     target = result['_target']
 966                     outcome = result['_outcome']
 967                     base_outcome = self._base_board_dict[target]
 968                     for fname in outcome.func_sizes:
 969                         self.PrintFuncSizeDetail(fname,
 970                                                  base_outcome.func_sizes[fname],
 971                                                  outcome.func_sizes[fname])
 972
 973
 974     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 975                          show_bloat):
 976         """Print a summary of image sizes broken down by section.
 977
 978         The summary takes the form of one line per architecture. The
 979         line contains deltas for each of the sections (+ means the section
 980         got bigger, - means smaller). The numbers are the average number
 981         of bytes that a board in this section increased by.
 982
 983         For example:
 984            powerpc: (622 boards)   text -0.0
 985           arm: (285 boards)   text -0.0
 986           nds32: (3 boards)   text -8.0
 987
 988         Args:
 989             board_selected: Dict containing boards to summarise, keyed by
 990                 board.target
 991             board_dict: Dict containing boards for which we built this
 992                 commit, keyed by board.target. The value is an Outcome object.
 993             show_detail: Show size delta detail for each board
 994             show_bloat: Show detail for each function
 995         """
 996         arch_list = {}
 997         arch_count = {}
 998
 999         # Calculate changes in size for different image parts
1000         # The previous sizes are in Board.sizes, for each board
1001         for target in board_dict:
1002             if target not in board_selected:
1003                 continue
1004             base_sizes = self._base_board_dict[target].sizes
1005             outcome = board_dict[target]
1006             sizes = outcome.sizes
1007
1008             # Loop through the list of images, creating a dict of size
1009             # changes for each image/part. We end up with something like
1010             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1011             # which means that U-Boot data increased by 5 bytes and SPL
1012             # text decreased by 4.
1013             err = {'_target' : target}
1014             for image in sizes:
1015                 if image in base_sizes:
1016                     base_image = base_sizes[image]
1017                     # Loop through the text, data, bss parts
1018                     for part in sorted(sizes[image]):
1019                         diff = sizes[image][part] - base_image[part]
1020                         col = None
1021                         if diff:
1022                             if image == 'u-boot':
1023                                 name = part
1024                             else:
1025                                 name = image + ':' + part
1026                             err[name] = diff
1027             arch = board_selected[target].arch
1028             if not arch in arch_count:
1029                 arch_count[arch] = 1
1030             else:
1031                 arch_count[arch] += 1
1032             if not sizes:
1033                 pass    # Only add to our list when we have some stats
1034             elif not arch in arch_list:
1035                 arch_list[arch] = [err]
1036             else:
1037                 arch_list[arch].append(err)
1038
1039         # We now have a list of image size changes sorted by arch
1040         # Print out a summary of these
1041         for arch, target_list in arch_list.items():
1042             # Get total difference for each type
1043             totals = {}
1044             for result in target_list:
1045                 total = 0
1046                 for name, diff in result.items():
1047                     if name.startswith('_'):
1048                         continue
1049                     total += diff
1050                     if name in totals:
1051                         totals[name] += diff
1052                     else:
1053                         totals[name] = diff
1054                 result['_total_diff'] = total
1055                 result['_outcome'] = board_dict[result['_target']]
1056
1057             count = len(target_list)
1058             printed_arch = False
1059             for name in sorted(totals):
1060                 diff = totals[name]
1061                 if diff:
1062                     # Display the average difference in this name for this
1063                     # architecture
1064                     avg_diff = float(diff) / count
1065                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1066                     msg = ' %s %+1.1f' % (name, avg_diff)
1067                     if not printed_arch:
1068                         Print('%10s: (for %d/%d boards)' % (arch, count,
1069                               arch_count[arch]), newline=False)
1070                         printed_arch = True
1071                     Print(msg, colour=color, newline=False)
1072
1073             if printed_arch:
1074                 Print()
1075                 if show_detail:
1076                     self.PrintSizeDetail(target_list, show_bloat)
1077
1078
1079     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1080                            err_line_boards, warn_lines, warn_line_boards,
1081                            config, environment, show_sizes, show_detail,
1082                            show_bloat, show_config, show_environment):
1083         """Compare results with the base results and display delta.
1084
1085         Only boards mentioned in board_selected will be considered. This
1086         function is intended to be called repeatedly with the results of
1087         each commit. It therefore shows a 'diff' between what it saw in
1088         the last call and what it sees now.
1089
1090         Args:
1091             board_selected: Dict containing boards to summarise, keyed by
1092                 board.target
1093             board_dict: Dict containing boards for which we built this
1094                 commit, keyed by board.target. The value is an Outcome object.
1095             err_lines: A list of errors for this commit, or [] if there is
1096                 none, or we don't want to print errors
1097             err_line_boards: Dict keyed by error line, containing a list of
1098                 the Board objects with that error
1099             warn_lines: A list of warnings for this commit, or [] if there is
1100                 none, or we don't want to print errors
1101             warn_line_boards: Dict keyed by warning line, containing a list of
1102                 the Board objects with that warning
1103             config: Dictionary keyed by filename - e.g. '.config'. Each
1104                     value is itself a dictionary:
1105                         key: config name
1106                         value: config value
1107             environment: Dictionary keyed by environment variable, Each
1108                      value is the value of environment variable.
1109             show_sizes: Show image size deltas
1110             show_detail: Show size delta detail for each board if show_sizes
1111             show_bloat: Show detail for each function
1112             show_config: Show config changes
1113             show_environment: Show environment changes
1114         """
1115         def _BoardList(line, line_boards):
1116             """Helper function to get a line of boards containing a line
1117
1118             Args:
1119                 line: Error line to search for
1120                 line_boards: boards to search, each a Board
1121             Return:
1122                 List of boards with that error line, or [] if the user has not
1123                     requested such a list
1124             """
1125             boards = []
1126             board_set = set()
1127             if self._list_error_boards:
1128                 for board in line_boards[line]:
1129                     if not board in board_set:
1130                         boards.append(board)
1131                         board_set.add(board)
1132             return boards
1133
1134         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1135                             char):
1136             """Calculate the required output based on changes in errors
1137
1138             Args:
1139                 base_lines: List of errors/warnings for previous commit
1140                 base_line_boards: Dict keyed by error line, containing a list
1141                     of the Board objects with that error in the previous commit
1142                 lines: List of errors/warning for this commit, each a str
1143                 line_boards: Dict keyed by error line, containing a list
1144                     of the Board objects with that error in this commit
1145                 char: Character representing error ('') or warning ('w'). The
1146                     broken ('+') or fixed ('-') characters are added in this
1147                     function
1148
1149             Returns:
1150                 Tuple
1151                     List of ErrLine objects for 'better' lines
1152                     List of ErrLine objects for 'worse' lines
1153             """
1154             better_lines = []
1155             worse_lines = []
1156             for line in lines:
1157                 if line not in base_lines:
1158                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1159                                       line)
1160                     worse_lines.append(errline)
1161             for line in base_lines:
1162                 if line not in lines:
1163                     errline = ErrLine(char + '-',
1164                                       _BoardList(line, base_line_boards), line)
1165                     better_lines.append(errline)
1166             return better_lines, worse_lines
1167
1168         def _CalcConfig(delta, name, config):
1169             """Calculate configuration changes
1170
1171             Args:
1172                 delta: Type of the delta, e.g. '+'
1173                 name: name of the file which changed (e.g. .config)
1174                 config: configuration change dictionary
1175                     key: config name
1176                     value: config value
1177             Returns:
1178                 String containing the configuration changes which can be
1179                     printed
1180             """
1181             out = ''
1182             for key in sorted(config.keys()):
1183                 out += '%s=%s ' % (key, config[key])
1184             return '%s %s: %s' % (delta, name, out)
1185
1186         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1187             """Add changes in configuration to a list
1188
1189             Args:
1190                 lines: list to add to
1191                 name: config file name
1192                 config_plus: configurations added, dictionary
1193                     key: config name
1194                     value: config value
1195                 config_minus: configurations removed, dictionary
1196                     key: config name
1197                     value: config value
1198                 config_change: configurations changed, dictionary
1199                     key: config name
1200                     value: config value
1201             """
1202             if config_plus:
1203                 lines.append(_CalcConfig('+', name, config_plus))
1204             if config_minus:
1205                 lines.append(_CalcConfig('-', name, config_minus))
1206             if config_change:
1207                 lines.append(_CalcConfig('c', name, config_change))
1208
1209         def _OutputConfigInfo(lines):
1210             for line in lines:
1211                 if not line:
1212                     continue
1213                 if line[0] == '+':
1214                     col = self.col.GREEN
1215                 elif line[0] == '-':
1216                     col = self.col.RED
1217                 elif line[0] == 'c':
1218                     col = self.col.YELLOW
1219                 Print('   ' + line, newline=True, colour=col)
1220
1221         def _OutputErrLines(err_lines, colour):
1222             """Output the line of error/warning lines, if not empty
1223
1224             Also increments self._error_lines if err_lines not empty
1225
1226             Args:
1227                 err_lines: List of ErrLine objects, each an error or warning
1228                     line, possibly including a list of boards with that
1229                     error/warning
1230                 colour: Colour to use for output
1231             """
1232             if err_lines:
1233                 out_list = []
1234                 for line in err_lines:
1235                     boards = ''
1236                     names = [board.target for board in line.boards]
1237                     board_str = ' '.join(names) if names else ''
1238                     if board_str:
1239                         out = self.col.Color(colour, line.char + '(')
1240                         out += self.col.Color(self.col.MAGENTA, board_str,
1241                                               bright=False)
1242                         out += self.col.Color(colour, ') %s' % line.errline)
1243                     else:
1244                         out = self.col.Color(colour, line.char + line.errline)
1245                     out_list.append(out)
1246                 Print('\n'.join(out_list))
1247                 self._error_lines += 1
1248
1249
1250         ok_boards = []      # List of boards fixed since last commit
1251         warn_boards = []    # List of boards with warnings since last commit
1252         err_boards = []     # List of new broken boards since last commit
1253         new_boards = []     # List of boards that didn't exist last time
1254         unknown_boards = [] # List of boards that were not built
1255
1256         for target in board_dict:
1257             if target not in board_selected:
1258                 continue
1259
1260             # If the board was built last time, add its outcome to a list
1261             if target in self._base_board_dict:
1262                 base_outcome = self._base_board_dict[target].rc
1263                 outcome = board_dict[target]
1264                 if outcome.rc == OUTCOME_UNKNOWN:
1265                     unknown_boards.append(target)
1266                 elif outcome.rc < base_outcome:
1267                     if outcome.rc == OUTCOME_WARNING:
1268                         warn_boards.append(target)
1269                     else:
1270                         ok_boards.append(target)
1271                 elif outcome.rc > base_outcome:
1272                     if outcome.rc == OUTCOME_WARNING:
1273                         warn_boards.append(target)
1274                     else:
1275                         err_boards.append(target)
1276             else:
1277                 new_boards.append(target)
1278
1279         # Get a list of errors and warnings that have appeared, and disappeared
1280         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1281                 self._base_err_line_boards, err_lines, err_line_boards, '')
1282         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1283                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1284
1285         # Display results by arch
1286         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1287                 worse_err, better_err, worse_warn, better_warn)):
1288             arch_list = {}
1289             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1290                     self.col.GREEN)
1291             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1292                     self.col.YELLOW)
1293             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1294                     self.col.RED)
1295             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1296             if self._show_unknown:
1297                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1298                         self.col.MAGENTA)
1299             for arch, target_list in arch_list.items():
1300                 Print('%10s: %s' % (arch, target_list))
1301                 self._error_lines += 1
1302             _OutputErrLines(better_err, colour=self.col.GREEN)
1303             _OutputErrLines(worse_err, colour=self.col.RED)
1304             _OutputErrLines(better_warn, colour=self.col.CYAN)
1305             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1306
1307         if show_sizes:
1308             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1309                                   show_bloat)
1310
1311         if show_environment and self._base_environment:
1312             lines = []
1313
1314             for target in board_dict:
1315                 if target not in board_selected:
1316                     continue
1317
1318                 tbase = self._base_environment[target]
1319                 tenvironment = environment[target]
1320                 environment_plus = {}
1321                 environment_minus = {}
1322                 environment_change = {}
1323                 base = tbase.environment
1324                 for key, value in tenvironment.environment.items():
1325                     if key not in base:
1326                         environment_plus[key] = value
1327                 for key, value in base.items():
1328                     if key not in tenvironment.environment:
1329                         environment_minus[key] = value
1330                 for key, value in base.items():
1331                     new_value = tenvironment.environment.get(key)
1332                     if new_value and value != new_value:
1333                         desc = '%s -> %s' % (value, new_value)
1334                         environment_change[key] = desc
1335
1336                 _AddConfig(lines, target, environment_plus, environment_minus,
1337                            environment_change)
1338
1339             _OutputConfigInfo(lines)
1340
1341         if show_config and self._base_config:
1342             summary = {}
1343             arch_config_plus = {}
1344             arch_config_minus = {}
1345             arch_config_change = {}
1346             arch_list = []
1347
1348             for target in board_dict:
1349                 if target not in board_selected:
1350                     continue
1351                 arch = board_selected[target].arch
1352                 if arch not in arch_list:
1353                     arch_list.append(arch)
1354
1355             for arch in arch_list:
1356                 arch_config_plus[arch] = {}
1357                 arch_config_minus[arch] = {}
1358                 arch_config_change[arch] = {}
1359                 for name in self.config_filenames:
1360                     arch_config_plus[arch][name] = {}
1361                     arch_config_minus[arch][name] = {}
1362                     arch_config_change[arch][name] = {}
1363
1364             for target in board_dict:
1365                 if target not in board_selected:
1366                     continue
1367
1368                 arch = board_selected[target].arch
1369
1370                 all_config_plus = {}
1371                 all_config_minus = {}
1372                 all_config_change = {}
1373                 tbase = self._base_config[target]
1374                 tconfig = config[target]
1375                 lines = []
1376                 for name in self.config_filenames:
1377                     if not tconfig.config[name]:
1378                         continue
1379                     config_plus = {}
1380                     config_minus = {}
1381                     config_change = {}
1382                     base = tbase.config[name]
1383                     for key, value in tconfig.config[name].items():
1384                         if key not in base:
1385                             config_plus[key] = value
1386                             all_config_plus[key] = value
1387                     for key, value in base.items():
1388                         if key not in tconfig.config[name]:
1389                             config_minus[key] = value
1390                             all_config_minus[key] = value
1391                     for key, value in base.items():
1392                         new_value = tconfig.config.get(key)
1393                         if new_value and value != new_value:
1394                             desc = '%s -> %s' % (value, new_value)
1395                             config_change[key] = desc
1396                             all_config_change[key] = desc
1397
1398                     arch_config_plus[arch][name].update(config_plus)
1399                     arch_config_minus[arch][name].update(config_minus)
1400                     arch_config_change[arch][name].update(config_change)
1401
1402                     _AddConfig(lines, name, config_plus, config_minus,
1403                                config_change)
1404                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1405                            all_config_change)
1406                 summary[target] = '\n'.join(lines)
1407
1408             lines_by_target = {}
1409             for target, lines in summary.items():
1410                 if lines in lines_by_target:
1411                     lines_by_target[lines].append(target)
1412                 else:
1413                     lines_by_target[lines] = [target]
1414
1415             for arch in arch_list:
1416                 lines = []
1417                 all_plus = {}
1418                 all_minus = {}
1419                 all_change = {}
1420                 for name in self.config_filenames:
1421                     all_plus.update(arch_config_plus[arch][name])
1422                     all_minus.update(arch_config_minus[arch][name])
1423                     all_change.update(arch_config_change[arch][name])
1424                     _AddConfig(lines, name, arch_config_plus[arch][name],
1425                                arch_config_minus[arch][name],
1426                                arch_config_change[arch][name])
1427                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1428                 #arch_summary[target] = '\n'.join(lines)
1429                 if lines:
1430                     Print('%s:' % arch)
1431                     _OutputConfigInfo(lines)
1432
1433             for lines, targets in lines_by_target.items():
1434                 if not lines:
1435                     continue
1436                 Print('%s :' % ' '.join(sorted(targets)))
1437                 _OutputConfigInfo(lines.split('\n'))
1438
1439
1440         # Save our updated information for the next call to this function
1441         self._base_board_dict = board_dict
1442         self._base_err_lines = err_lines
1443         self._base_warn_lines = warn_lines
1444         self._base_err_line_boards = err_line_boards
1445         self._base_warn_line_boards = warn_line_boards
1446         self._base_config = config
1447         self._base_environment = environment
1448
1449         # Get a list of boards that did not get built, if needed
1450         not_built = []
1451         for board in board_selected:
1452             if not board in board_dict:
1453                 not_built.append(board)
1454         if not_built:
1455             Print("Boards not built (%d): %s" % (len(not_built),
1456                   ', '.join(not_built)))
1457
1458     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1459             (board_dict, err_lines, err_line_boards, warn_lines,
1460              warn_line_boards, config, environment) = self.GetResultSummary(
1461                     board_selected, commit_upto,
1462                     read_func_sizes=self._show_bloat,
1463                     read_config=self._show_config,
1464                     read_environment=self._show_environment)
1465             if commits:
1466                 msg = '%02d: %s' % (commit_upto + 1,
1467                         commits[commit_upto].subject)
1468                 Print(msg, colour=self.col.BLUE)
1469             self.PrintResultSummary(board_selected, board_dict,
1470                     err_lines if self._show_errors else [], err_line_boards,
1471                     warn_lines if self._show_errors else [], warn_line_boards,
1472                     config, environment, self._show_sizes, self._show_detail,
1473                     self._show_bloat, self._show_config, self._show_environment)
1474
1475     def ShowSummary(self, commits, board_selected):
1476         """Show a build summary for U-Boot for a given board list.
1477
1478         Reset the result summary, then repeatedly call GetResultSummary on
1479         each commit's results, then display the differences we see.
1480
1481         Args:
1482             commit: Commit objects to summarise
1483             board_selected: Dict containing boards to summarise
1484         """
1485         self.commit_count = len(commits) if commits else 1
1486         self.commits = commits
1487         self.ResetResultSummary(board_selected)
1488         self._error_lines = 0
1489
1490         for commit_upto in range(0, self.commit_count, self._step):
1491             self.ProduceResultSummary(commit_upto, commits, board_selected)
1492         if not self._error_lines:
1493             Print('(no errors to report)', colour=self.col.GREEN)
1494
1495
1496     def SetupBuild(self, board_selected, commits):
1497         """Set up ready to start a build.
1498
1499         Args:
1500             board_selected: Selected boards to build
1501             commits: Selected commits to build
1502         """
1503         # First work out how many commits we will build
1504         count = (self.commit_count + self._step - 1) // self._step
1505         self.count = len(board_selected) * count
1506         self.upto = self.warned = self.fail = 0
1507         self._timestamps = collections.deque()
1508
1509     def GetThreadDir(self, thread_num):
1510         """Get the directory path to the working dir for a thread.
1511
1512         Args:
1513             thread_num: Number of thread to check.
1514         """
1515         if self.work_in_output:
1516             return self._working_dir
1517         return os.path.join(self._working_dir, '%02d' % thread_num)
1518
1519     def _PrepareThread(self, thread_num, setup_git):
1520         """Prepare the working directory for a thread.
1521
1522         This clones or fetches the repo into the thread's work directory.
1523
1524         Args:
1525             thread_num: Thread number (0, 1, ...)
1526             setup_git: True to set up a git repo clone
1527         """
1528         thread_dir = self.GetThreadDir(thread_num)
1529         builderthread.Mkdir(thread_dir)
1530         git_dir = os.path.join(thread_dir, '.git')
1531
1532         # Clone the repo if it doesn't already exist
1533         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1534         # we have a private index but uses the origin repo's contents?
1535         if setup_git and self.git_dir:
1536             src_dir = os.path.abspath(self.git_dir)
1537             if os.path.exists(git_dir):
1538                 Print('\rFetching repo for thread %d' % thread_num,
1539                       newline=False)
1540                 gitutil.Fetch(git_dir, thread_dir)
1541                 terminal.PrintClear()
1542             else:
1543                 Print('\rCloning repo for thread %d' % thread_num,
1544                       newline=False)
1545                 gitutil.Clone(src_dir, thread_dir)
1546                 terminal.PrintClear()
1547
1548     def _PrepareWorkingSpace(self, max_threads, setup_git):
1549         """Prepare the working directory for use.
1550
1551         Set up the git repo for each thread.
1552
1553         Args:
1554             max_threads: Maximum number of threads we expect to need.
1555             setup_git: True to set up a git repo clone
1556         """
1557         builderthread.Mkdir(self._working_dir)
1558         for thread in range(max_threads):
1559             self._PrepareThread(thread, setup_git)
1560
1561     def _GetOutputSpaceRemovals(self):
1562         """Get the output directories ready to receive files.
1563
1564         Figure out what needs to be deleted in the output directory before it
1565         can be used. We only delete old buildman directories which have the
1566         expected name pattern. See _GetOutputDir().
1567
1568         Returns:
1569             List of full paths of directories to remove
1570         """
1571         if not self.commits:
1572             return
1573         dir_list = []
1574         for commit_upto in range(self.commit_count):
1575             dir_list.append(self._GetOutputDir(commit_upto))
1576
1577         to_remove = []
1578         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1579             if dirname not in dir_list:
1580                 leaf = dirname[len(self.base_dir) + 1:]
1581                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1582                 if m:
1583                     to_remove.append(dirname)
1584         return to_remove
1585
1586     def _PrepareOutputSpace(self):
1587         """Get the output directories ready to receive files.
1588
1589         We delete any output directories which look like ones we need to
1590         create. Having left over directories is confusing when the user wants
1591         to check the output manually.
1592         """
1593         to_remove = self._GetOutputSpaceRemovals()
1594         if to_remove:
1595             Print('Removing %d old build directories...' % len(to_remove),
1596                   newline=False)
1597             for dirname in to_remove:
1598                 shutil.rmtree(dirname)
1599             terminal.PrintClear()
1600
1601     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1602         """Build all commits for a list of boards
1603
1604         Args:
1605             commits: List of commits to be build, each a Commit object
1606             boards_selected: Dict of selected boards, key is target name,
1607                     value is Board object
1608             keep_outputs: True to save build output files
1609             verbose: Display build results as they are completed
1610         Returns:
1611             Tuple containing:
1612                 - number of boards that failed to build
1613                 - number of boards that issued warnings
1614         """
1615         self.commit_count = len(commits) if commits else 1
1616         self.commits = commits
1617         self._verbose = verbose
1618
1619         self.ResetResultSummary(board_selected)
1620         builderthread.Mkdir(self.base_dir, parents = True)
1621         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1622                 commits is not None)
1623         self._PrepareOutputSpace()
1624         Print('\rStarting build...', newline=False)
1625         self.SetupBuild(board_selected, commits)
1626         self.ProcessResult(None)
1627
1628         # Create jobs to build all commits for each board
1629         for brd in board_selected.values():
1630             job = builderthread.BuilderJob()
1631             job.board = brd
1632             job.commits = commits
1633             job.keep_outputs = keep_outputs
1634             job.work_in_output = self.work_in_output
1635             job.step = self._step
1636             self.queue.put(job)
1637
1638         term = threading.Thread(target=self.queue.join)
1639         term.setDaemon(True)
1640         term.start()
1641         while term.isAlive():
1642             term.join(100)
1643
1644         # Wait until we have processed all output
1645         self.out_queue.join()
1646         Print()
1647
1648         msg = 'Completed: %d total built' % self.count
1649         if self.already_done:
1650            msg += ' (%d previously' % self.already_done
1651            if self.already_done != self.count:
1652                msg += ', %d newly' % (self.count - self.already_done)
1653            msg += ')'
1654         duration = datetime.now() - self._start_time
1655         if duration > timedelta(microseconds=1000000):
1656             if duration.microseconds >= 500000:
1657                 duration = duration + timedelta(seconds=1)
1658             duration = duration - timedelta(microseconds=duration.microseconds)
1659             msg += ', duration %s' % duration
1660         Print(msg)
1661
1662         return (self.fail, self.warned)