tools/buildman/builder.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
   5 #
   6
   7 import collections
   8 from datetime import datetime, timedelta
   9 import glob
  10 import os
  11 import re
  12 import queue
  13 import shutil
  14 import signal
  15 import string
  16 import sys
  17 import threading
  18 import time
  19
  20 import builderthread
  21 import command
  22 import gitutil
  23 import terminal
  24 from terminal import Print
  25 import toolchain
  26
  27 """
  28 Theory of Operation
  29
  30 Please see README for user documentation, and you should be familiar with
  31 that before trying to make sense of this.
  32
  33 Buildman works by keeping the machine as busy as possible, building different
  34 commits for different boards on multiple CPUs at once.
  35
  36 The source repo (self.git_dir) contains all the commits to be built. Each
  37 thread works on a single board at a time. It checks out the first commit,
  38 configures it for that board, then builds it. Then it checks out the next
  39 commit and builds it (typically without re-configuring). When it runs out
  40 of commits, it gets another job from the builder and starts again with that
  41 board.
  42
  43 Clearly the builder threads could work either way - they could check out a
  44 commit and then built it for all boards. Using separate directories for each
  45 commit/board pair they could leave their build product around afterwards
  46 also.
  47
  48 The intent behind building a single board for multiple commits, is to make
  49 use of incremental builds. Since each commit is built incrementally from
  50 the previous one, builds are faster. Reconfiguring for a different board
  51 removes all intermediate object files.
  52
  53 Many threads can be working at once, but each has its own working directory.
  54 When a thread finishes a build, it puts the output files into a result
  55 directory.
  56
  57 The base directory used by buildman is normally '../<branch>', i.e.
  58 a directory higher than the source repository and named after the branch
  59 being built.
  60
  61 Within the base directory, we have one subdirectory for each commit. Within
  62 that is one subdirectory for each board. Within that is the build output for
  63 that commit/board combination.
  64
  65 Buildman also create working directories for each thread, in a .bm-work/
  66 subdirectory in the base dir.
  67
  68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
  69 'seaboard', and say that us-net has two commits. We will have directories
  70 like this:
  71
  72 us-net/             base directory
  73     01_of_02_g4ed4ebc_net--Add-tftp-speed-/
  74         sandbox/
  75             u-boot.bin
  76         seaboard/
  77             u-boot.bin
  78     02_of_02_g4ed4ebc_net--Check-tftp-comp/
  79         sandbox/
  80             u-boot.bin
  81         seaboard/
  82             u-boot.bin
  83     .bm-work/
  84         00/         working directory for thread 0 (contains source checkout)
  85             build/  build output
  86         01/         working directory for thread 1
  87             build/  build output
  88         ...
  89 u-boot/             source directory
  90     .git/           repository
  91 """
  92
  93 """Holds information about a particular error line we are outputing
  94
  95    char: Character representation: '+': error, '-': fixed error, 'w+': warning,
  96        'w-' = fixed warning
  97    boards: List of Board objects which have line in the error/warning output
  98    errline: The text of the error line
  99 """
 100 ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
 101
 102 # Possible build outcomes
 103 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 104
 105 # Translate a commit subject into a valid filename (and handle unicode)
 106 trans_valid_chars = str.maketrans('/: ', '---')
 107
 108 BASE_CONFIG_FILENAMES = [
 109     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
 110 ]
 111
 112 EXTRA_CONFIG_FILENAMES = [
 113     '.config', '.config-spl', '.config-tpl',
 114     'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
 115     'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
 116 ]
 117
 118 class Config:
 119     """Holds information about configuration settings for a board."""
 120     def __init__(self, config_filename, target):
 121         self.target = target
 122         self.config = {}
 123         for fname in config_filename:
 124             self.config[fname] = {}
 125
 126     def Add(self, fname, key, value):
 127         self.config[fname][key] = value
 128
 129     def __hash__(self):
 130         val = 0
 131         for fname in self.config:
 132             for key, value in self.config[fname].items():
 133                 print(key, value)
 134                 val = val ^ hash(key) & hash(value)
 135         return val
 136
 137 class Environment:
 138     """Holds information about environment variables for a board."""
 139     def __init__(self, target):
 140         self.target = target
 141         self.environment = {}
 142
 143     def Add(self, key, value):
 144         self.environment[key] = value
 145
 146 class Builder:
 147     """Class for building U-Boot for a particular commit.
 148
 149     Public members: (many should ->private)
 150         already_done: Number of builds already completed
 151         base_dir: Base directory to use for builder
 152         checkout: True to check out source, False to skip that step.
 153             This is used for testing.
 154         col: terminal.Color() object
 155         count: Number of commits to build
 156         do_make: Method to call to invoke Make
 157         fail: Number of builds that failed due to error
 158         force_build: Force building even if a build already exists
 159         force_config_on_failure: If a commit fails for a board, disable
 160             incremental building for the next commit we build for that
 161             board, so that we will see all warnings/errors again.
 162         force_build_failures: If a previously-built build (i.e. built on
 163             a previous run of buildman) is marked as failed, rebuild it.
 164         git_dir: Git directory containing source repository
 165         num_jobs: Number of jobs to run at once (passed to make as -j)
 166         num_threads: Number of builder threads to run
 167         out_queue: Queue of results to process
 168         re_make_err: Compiled regular expression for ignore_lines
 169         queue: Queue of jobs to run
 170         threads: List of active threads
 171         toolchains: Toolchains object to use for building
 172         upto: Current commit number we are building (0.count-1)
 173         warned: Number of builds that produced at least one warning
 174         force_reconfig: Reconfigure U-Boot on each comiit. This disables
 175             incremental building, where buildman reconfigures on the first
 176             commit for a baord, and then just does an incremental build for
 177             the following commits. In fact buildman will reconfigure and
 178             retry for any failing commits, so generally the only effect of
 179             this option is to slow things down.
 180         in_tree: Build U-Boot in-tree instead of specifying an output
 181             directory separate from the source code. This option is really
 182             only useful for testing in-tree builds.
 183         work_in_output: Use the output directory as the work directory and
 184             don't write to a separate output directory.
 185
 186     Private members:
 187         _base_board_dict: Last-summarised Dict of boards
 188         _base_err_lines: Last-summarised list of errors
 189         _base_warn_lines: Last-summarised list of warnings
 190         _build_period_us: Time taken for a single build (float object).
 191         _complete_delay: Expected delay until completion (timedelta)
 192         _next_delay_update: Next time we plan to display a progress update
 193                 (datatime)
 194         _show_unknown: Show unknown boards (those not built) in summary
 195         _start_time: Start time for the build
 196         _timestamps: List of timestamps for the completion of the last
 197             last _timestamp_count builds. Each is a datetime object.
 198         _timestamp_count: Number of timestamps to keep in our list.
 199         _working_dir: Base working directory containing all threads
 200     """
 201     class Outcome:
 202         """Records a build outcome for a single make invocation
 203
 204         Public Members:
 205             rc: Outcome value (OUTCOME_...)
 206             err_lines: List of error lines or [] if none
 207             sizes: Dictionary of image size information, keyed by filename
 208                 - Each value is itself a dictionary containing
 209                     values for 'text', 'data' and 'bss', being the integer
 210                     size in bytes of each section.
 211             func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
 212                     value is itself a dictionary:
 213                         key: function name
 214                         value: Size of function in bytes
 215             config: Dictionary keyed by filename - e.g. '.config'. Each
 216                     value is itself a dictionary:
 217                         key: config name
 218                         value: config value
 219             environment: Dictionary keyed by environment variable, Each
 220                      value is the value of environment variable.
 221         """
 222         def __init__(self, rc, err_lines, sizes, func_sizes, config,
 223                      environment):
 224             self.rc = rc
 225             self.err_lines = err_lines
 226             self.sizes = sizes
 227             self.func_sizes = func_sizes
 228             self.config = config
 229             self.environment = environment
 230
 231     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
 232                  gnu_make='make', checkout=True, show_unknown=True, step=1,
 233                  no_subdirs=False, full_path=False, verbose_build=False,
 234                  mrproper=False, per_board_out_dir=False,
 235                  config_only=False, squash_config_y=False,
 236                  warnings_as_errors=False, work_in_output=False):
 237         """Create a new Builder object
 238
 239         Args:
 240             toolchains: Toolchains object to use for building
 241             base_dir: Base directory to use for builder
 242             git_dir: Git directory containing source repository
 243             num_threads: Number of builder threads to run
 244             num_jobs: Number of jobs to run at once (passed to make as -j)
 245             gnu_make: the command name of GNU Make.
 246             checkout: True to check out source, False to skip that step.
 247                 This is used for testing.
 248             show_unknown: Show unknown boards (those not built) in summary
 249             step: 1 to process every commit, n to process every nth commit
 250             no_subdirs: Don't create subdirectories when building current
 251                 source for a single board
 252             full_path: Return the full path in CROSS_COMPILE and don't set
 253                 PATH
 254             verbose_build: Run build with V=1 and don't use 'make -s'
 255             mrproper: Always run 'make mrproper' when configuring
 256             per_board_out_dir: Build in a separate persistent directory per
 257                 board rather than a thread-specific directory
 258             config_only: Only configure each build, don't build it
 259             squash_config_y: Convert CONFIG options with the value 'y' to '1'
 260             warnings_as_errors: Treat all compiler warnings as errors
 261             work_in_output: Use the output directory as the work directory and
 262                 don't write to a separate output directory.
 263         """
 264         self.toolchains = toolchains
 265         self.base_dir = base_dir
 266         if work_in_output:
 267             self._working_dir = base_dir
 268         else:
 269             self._working_dir = os.path.join(base_dir, '.bm-work')
 270         self.threads = []
 271         self.do_make = self.Make
 272         self.gnu_make = gnu_make
 273         self.checkout = checkout
 274         self.num_threads = num_threads
 275         self.num_jobs = num_jobs
 276         self.already_done = 0
 277         self.force_build = False
 278         self.git_dir = git_dir
 279         self._show_unknown = show_unknown
 280         self._timestamp_count = 10
 281         self._build_period_us = None
 282         self._complete_delay = None
 283         self._next_delay_update = datetime.now()
 284         self._start_time = datetime.now()
 285         self.force_config_on_failure = True
 286         self.force_build_failures = False
 287         self.force_reconfig = False
 288         self._step = step
 289         self.in_tree = False
 290         self._error_lines = 0
 291         self.no_subdirs = no_subdirs
 292         self.full_path = full_path
 293         self.verbose_build = verbose_build
 294         self.config_only = config_only
 295         self.squash_config_y = squash_config_y
 296         self.config_filenames = BASE_CONFIG_FILENAMES
 297         self.work_in_output = work_in_output
 298         if not self.squash_config_y:
 299             self.config_filenames += EXTRA_CONFIG_FILENAMES
 300
 301         self.warnings_as_errors = warnings_as_errors
 302         self.col = terminal.Color()
 303
 304         self._re_function = re.compile('(.*): In function.*')
 305         self._re_files = re.compile('In file included from.*')
 306         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
 307         self._re_dtb_warning = re.compile('(.*): Warning .*')
 308         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
 309
 310         self.queue = queue.Queue()
 311         self.out_queue = queue.Queue()
 312         for i in range(self.num_threads):
 313             t = builderthread.BuilderThread(self, i, mrproper,
 314                     per_board_out_dir)
 315             t.setDaemon(True)
 316             t.start()
 317             self.threads.append(t)
 318
 319         t = builderthread.ResultThread(self)
 320         t.setDaemon(True)
 321         t.start()
 322         self.threads.append(t)
 323
 324         ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
 325         self.re_make_err = re.compile('|'.join(ignore_lines))
 326
 327         # Handle existing graceful with SIGINT / Ctrl-C
 328         signal.signal(signal.SIGINT, self.signal_handler)
 329
 330     def __del__(self):
 331         """Get rid of all threads created by the builder"""
 332         for t in self.threads:
 333             del t
 334
 335     def signal_handler(self, signal, frame):
 336         sys.exit(1)
 337
 338     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
 339                           show_detail=False, show_bloat=False,
 340                           list_error_boards=False, show_config=False,
 341                           show_environment=False):
 342         """Setup display options for the builder.
 343
 344         show_errors: True to show summarised error/warning info
 345         show_sizes: Show size deltas
 346         show_detail: Show size delta detail for each board if show_sizes
 347         show_bloat: Show detail for each function
 348         list_error_boards: Show the boards which caused each error/warning
 349         show_config: Show config deltas
 350         show_environment: Show environment deltas
 351         """
 352         self._show_errors = show_errors
 353         self._show_sizes = show_sizes
 354         self._show_detail = show_detail
 355         self._show_bloat = show_bloat
 356         self._list_error_boards = list_error_boards
 357         self._show_config = show_config
 358         self._show_environment = show_environment
 359
 360     def _AddTimestamp(self):
 361         """Add a new timestamp to the list and record the build period.
 362
 363         The build period is the length of time taken to perform a single
 364         build (one board, one commit).
 365         """
 366         now = datetime.now()
 367         self._timestamps.append(now)
 368         count = len(self._timestamps)
 369         delta = self._timestamps[-1] - self._timestamps[0]
 370         seconds = delta.total_seconds()
 371
 372         # If we have enough data, estimate build period (time taken for a
 373         # single build) and therefore completion time.
 374         if count > 1 and self._next_delay_update < now:
 375             self._next_delay_update = now + timedelta(seconds=2)
 376             if seconds > 0:
 377                 self._build_period = float(seconds) / count
 378                 todo = self.count - self.upto
 379                 self._complete_delay = timedelta(microseconds=
 380                         self._build_period * todo * 1000000)
 381                 # Round it
 382                 self._complete_delay -= timedelta(
 383                         microseconds=self._complete_delay.microseconds)
 384
 385         if seconds > 60:
 386             self._timestamps.popleft()
 387             count -= 1
 388
 389     def SelectCommit(self, commit, checkout=True):
 390         """Checkout the selected commit for this build
 391         """
 392         self.commit = commit
 393         if checkout and self.checkout:
 394             gitutil.Checkout(commit.hash)
 395
 396     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
 397         """Run make
 398
 399         Args:
 400             commit: Commit object that is being built
 401             brd: Board object that is being built
 402             stage: Stage that we are at (mrproper, config, build)
 403             cwd: Directory where make should be run
 404             args: Arguments to pass to make
 405             kwargs: Arguments to pass to command.RunPipe()
 406         """
 407         cmd = [self.gnu_make] + list(args)
 408         result = command.RunPipe([cmd], capture=True, capture_stderr=True,
 409                 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
 410         if self.verbose_build:
 411             result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
 412             result.combined = '%s\n' % (' '.join(cmd)) + result.combined
 413         return result
 414
 415     def ProcessResult(self, result):
 416         """Process the result of a build, showing progress information
 417
 418         Args:
 419             result: A CommandResult object, which indicates the result for
 420                     a single build
 421         """
 422         col = terminal.Color()
 423         if result:
 424             target = result.brd.target
 425
 426             self.upto += 1
 427             if result.return_code != 0:
 428                 self.fail += 1
 429             elif result.stderr:
 430                 self.warned += 1
 431             if result.already_done:
 432                 self.already_done += 1
 433             if self._verbose:
 434                 terminal.PrintClear()
 435                 boards_selected = {target : result.brd}
 436                 self.ResetResultSummary(boards_selected)
 437                 self.ProduceResultSummary(result.commit_upto, self.commits,
 438                                           boards_selected)
 439         else:
 440             target = '(starting)'
 441
 442         # Display separate counts for ok, warned and fail
 443         ok = self.upto - self.warned - self.fail
 444         line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
 445         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
 446         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 447
 448         line += ' /%-5d  ' % self.count
 449         remaining = self.count - self.upto
 450         if remaining:
 451             line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
 452         else:
 453             line += ' ' * 8
 454
 455         # Add our current completion time estimate
 456         self._AddTimestamp()
 457         if self._complete_delay:
 458             line += '%s  : ' % self._complete_delay
 459
 460         line += target
 461         terminal.PrintClear()
 462         Print(line, newline=False, limit_to_line=True)
 463
 464     def _GetOutputDir(self, commit_upto):
 465         """Get the name of the output directory for a commit number
 466
 467         The output directory is typically .../<branch>/<commit>.
 468
 469         Args:
 470             commit_upto: Commit number to use (0..self.count-1)
 471         """
 472         commit_dir = None
 473         if self.commits:
 474             commit = self.commits[commit_upto]
 475             subject = commit.subject.translate(trans_valid_chars)
 476             # See _GetOutputSpaceRemovals() which parses this name
 477             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
 478                     self.commit_count, commit.hash, subject[:20]))
 479         elif not self.no_subdirs:
 480             commit_dir = 'current'
 481         if not commit_dir:
 482             return self.base_dir
 483         return os.path.join(self.base_dir, commit_dir)
 484
 485     def GetBuildDir(self, commit_upto, target):
 486         """Get the name of the build directory for a commit number
 487
 488         The build directory is typically .../<branch>/<commit>/<target>.
 489
 490         Args:
 491             commit_upto: Commit number to use (0..self.count-1)
 492             target: Target name
 493         """
 494         output_dir = self._GetOutputDir(commit_upto)
 495         return os.path.join(output_dir, target)
 496
 497     def GetDoneFile(self, commit_upto, target):
 498         """Get the name of the done file for a commit number
 499
 500         Args:
 501             commit_upto: Commit number to use (0..self.count-1)
 502             target: Target name
 503         """
 504         return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
 505
 506     def GetSizesFile(self, commit_upto, target):
 507         """Get the name of the sizes file for a commit number
 508
 509         Args:
 510             commit_upto: Commit number to use (0..self.count-1)
 511             target: Target name
 512         """
 513         return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
 514
 515     def GetFuncSizesFile(self, commit_upto, target, elf_fname):
 516         """Get the name of the funcsizes file for a commit number and ELF file
 517
 518         Args:
 519             commit_upto: Commit number to use (0..self.count-1)
 520             target: Target name
 521             elf_fname: Filename of elf image
 522         """
 523         return os.path.join(self.GetBuildDir(commit_upto, target),
 524                             '%s.sizes' % elf_fname.replace('/', '-'))
 525
 526     def GetObjdumpFile(self, commit_upto, target, elf_fname):
 527         """Get the name of the objdump file for a commit number and ELF file
 528
 529         Args:
 530             commit_upto: Commit number to use (0..self.count-1)
 531             target: Target name
 532             elf_fname: Filename of elf image
 533         """
 534         return os.path.join(self.GetBuildDir(commit_upto, target),
 535                             '%s.objdump' % elf_fname.replace('/', '-'))
 536
 537     def GetErrFile(self, commit_upto, target):
 538         """Get the name of the err file for a commit number
 539
 540         Args:
 541             commit_upto: Commit number to use (0..self.count-1)
 542             target: Target name
 543         """
 544         output_dir = self.GetBuildDir(commit_upto, target)
 545         return os.path.join(output_dir, 'err')
 546
 547     def FilterErrors(self, lines):
 548         """Filter out errors in which we have no interest
 549
 550         We should probably use map().
 551
 552         Args:
 553             lines: List of error lines, each a string
 554         Returns:
 555             New list with only interesting lines included
 556         """
 557         out_lines = []
 558         for line in lines:
 559             if not self.re_make_err.search(line):
 560                 out_lines.append(line)
 561         return out_lines
 562
 563     def ReadFuncSizes(self, fname, fd):
 564         """Read function sizes from the output of 'nm'
 565
 566         Args:
 567             fd: File containing data to read
 568             fname: Filename we are reading from (just for errors)
 569
 570         Returns:
 571             Dictionary containing size of each function in bytes, indexed by
 572             function name.
 573         """
 574         sym = {}
 575         for line in fd.readlines():
 576             try:
 577                 if line.strip():
 578                     size, type, name = line[:-1].split()
 579             except:
 580                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
 581                 continue
 582             if type in 'tTdDbB':
 583                 # function names begin with '.' on 64-bit powerpc
 584                 if '.' in name[1:]:
 585                     name = 'static.' + name.split('.')[0]
 586                 sym[name] = sym.get(name, 0) + int(size, 16)
 587         return sym
 588
 589     def _ProcessConfig(self, fname):
 590         """Read in a .config, autoconf.mk or autoconf.h file
 591
 592         This function handles all config file types. It ignores comments and
 593         any #defines which don't start with CONFIG_.
 594
 595         Args:
 596             fname: Filename to read
 597
 598         Returns:
 599             Dictionary:
 600                 key: Config name (e.g. CONFIG_DM)
 601                 value: Config value (e.g. 1)
 602         """
 603         config = {}
 604         if os.path.exists(fname):
 605             with open(fname) as fd:
 606                 for line in fd:
 607                     line = line.strip()
 608                     if line.startswith('#define'):
 609                         values = line[8:].split(' ', 1)
 610                         if len(values) > 1:
 611                             key, value = values
 612                         else:
 613                             key = values[0]
 614                             value = '1' if self.squash_config_y else ''
 615                         if not key.startswith('CONFIG_'):
 616                             continue
 617                     elif not line or line[0] in ['#', '*', '/']:
 618                         continue
 619                     else:
 620                         key, value = line.split('=', 1)
 621                     if self.squash_config_y and value == 'y':
 622                         value = '1'
 623                     config[key] = value
 624         return config
 625
 626     def _ProcessEnvironment(self, fname):
 627         """Read in a uboot.env file
 628
 629         This function reads in environment variables from a file.
 630
 631         Args:
 632             fname: Filename to read
 633
 634         Returns:
 635             Dictionary:
 636                 key: environment variable (e.g. bootlimit)
 637                 value: value of environment variable (e.g. 1)
 638         """
 639         environment = {}
 640         if os.path.exists(fname):
 641             with open(fname) as fd:
 642                 for line in fd.read().split('\0'):
 643                     try:
 644                         key, value = line.split('=', 1)
 645                         environment[key] = value
 646                     except ValueError:
 647                         # ignore lines we can't parse
 648                         pass
 649         return environment
 650
 651     def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
 652                         read_config, read_environment):
 653         """Work out the outcome of a build.
 654
 655         Args:
 656             commit_upto: Commit number to check (0..n-1)
 657             target: Target board to check
 658             read_func_sizes: True to read function size information
 659             read_config: True to read .config and autoconf.h files
 660             read_environment: True to read uboot.env files
 661
 662         Returns:
 663             Outcome object
 664         """
 665         done_file = self.GetDoneFile(commit_upto, target)
 666         sizes_file = self.GetSizesFile(commit_upto, target)
 667         sizes = {}
 668         func_sizes = {}
 669         config = {}
 670         environment = {}
 671         if os.path.exists(done_file):
 672             with open(done_file, 'r') as fd:
 673                 try:
 674                     return_code = int(fd.readline())
 675                 except ValueError:
 676                     # The file may be empty due to running out of disk space.
 677                     # Try a rebuild
 678                     return_code = 1
 679                 err_lines = []
 680                 err_file = self.GetErrFile(commit_upto, target)
 681                 if os.path.exists(err_file):
 682                     with open(err_file, 'r') as fd:
 683                         err_lines = self.FilterErrors(fd.readlines())
 684
 685                 # Decide whether the build was ok, failed or created warnings
 686                 if return_code:
 687                     rc = OUTCOME_ERROR
 688                 elif len(err_lines):
 689                     rc = OUTCOME_WARNING
 690                 else:
 691                     rc = OUTCOME_OK
 692
 693                 # Convert size information to our simple format
 694                 if os.path.exists(sizes_file):
 695                     with open(sizes_file, 'r') as fd:
 696                         for line in fd.readlines():
 697                             values = line.split()
 698                             rodata = 0
 699                             if len(values) > 6:
 700                                 rodata = int(values[6], 16)
 701                             size_dict = {
 702                                 'all' : int(values[0]) + int(values[1]) +
 703                                         int(values[2]),
 704                                 'text' : int(values[0]) - rodata,
 705                                 'data' : int(values[1]),
 706                                 'bss' : int(values[2]),
 707                                 'rodata' : rodata,
 708                             }
 709                             sizes[values[5]] = size_dict
 710
 711             if read_func_sizes:
 712                 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
 713                 for fname in glob.glob(pattern):
 714                     with open(fname, 'r') as fd:
 715                         dict_name = os.path.basename(fname).replace('.sizes',
 716                                                                     '')
 717                         func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
 718
 719             if read_config:
 720                 output_dir = self.GetBuildDir(commit_upto, target)
 721                 for name in self.config_filenames:
 722                     fname = os.path.join(output_dir, name)
 723                     config[name] = self._ProcessConfig(fname)
 724
 725             if read_environment:
 726                 output_dir = self.GetBuildDir(commit_upto, target)
 727                 fname = os.path.join(output_dir, 'uboot.env')
 728                 environment = self._ProcessEnvironment(fname)
 729
 730             return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
 731                                    environment)
 732
 733         return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
 734
 735     def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
 736                          read_config, read_environment):
 737         """Calculate a summary of the results of building a commit.
 738
 739         Args:
 740             board_selected: Dict containing boards to summarise
 741             commit_upto: Commit number to summarize (0..self.count-1)
 742             read_func_sizes: True to read function size information
 743             read_config: True to read .config and autoconf.h files
 744             read_environment: True to read uboot.env files
 745
 746         Returns:
 747             Tuple:
 748                 Dict containing boards which passed building this commit.
 749                     keyed by board.target
 750                 List containing a summary of error lines
 751                 Dict keyed by error line, containing a list of the Board
 752                     objects with that error
 753                 List containing a summary of warning lines
 754                 Dict keyed by error line, containing a list of the Board
 755                     objects with that warning
 756                 Dictionary keyed by board.target. Each value is a dictionary:
 757                     key: filename - e.g. '.config'
 758                     value is itself a dictionary:
 759                         key: config name
 760                         value: config value
 761                 Dictionary keyed by board.target. Each value is a dictionary:
 762                     key: environment variable
 763                     value: value of environment variable
 764         """
 765         def AddLine(lines_summary, lines_boards, line, board):
 766             line = line.rstrip()
 767             if line in lines_boards:
 768                 lines_boards[line].append(board)
 769             else:
 770                 lines_boards[line] = [board]
 771                 lines_summary.append(line)
 772
 773         board_dict = {}
 774         err_lines_summary = []
 775         err_lines_boards = {}
 776         warn_lines_summary = []
 777         warn_lines_boards = {}
 778         config = {}
 779         environment = {}
 780
 781         for board in boards_selected.values():
 782             outcome = self.GetBuildOutcome(commit_upto, board.target,
 783                                            read_func_sizes, read_config,
 784                                            read_environment)
 785             board_dict[board.target] = outcome
 786             last_func = None
 787             last_was_warning = False
 788             for line in outcome.err_lines:
 789                 if line:
 790                     if (self._re_function.match(line) or
 791                             self._re_files.match(line)):
 792                         last_func = line
 793                     else:
 794                         is_warning = (self._re_warning.match(line) or
 795                                       self._re_dtb_warning.match(line))
 796                         is_note = self._re_note.match(line)
 797                         if is_warning or (last_was_warning and is_note):
 798                             if last_func:
 799                                 AddLine(warn_lines_summary, warn_lines_boards,
 800                                         last_func, board)
 801                             AddLine(warn_lines_summary, warn_lines_boards,
 802                                     line, board)
 803                         else:
 804                             if last_func:
 805                                 AddLine(err_lines_summary, err_lines_boards,
 806                                         last_func, board)
 807                             AddLine(err_lines_summary, err_lines_boards,
 808                                     line, board)
 809                         last_was_warning = is_warning
 810                         last_func = None
 811             tconfig = Config(self.config_filenames, board.target)
 812             for fname in self.config_filenames:
 813                 if outcome.config:
 814                     for key, value in outcome.config[fname].items():
 815                         tconfig.Add(fname, key, value)
 816             config[board.target] = tconfig
 817
 818             tenvironment = Environment(board.target)
 819             if outcome.environment:
 820                 for key, value in outcome.environment.items():
 821                     tenvironment.Add(key, value)
 822             environment[board.target] = tenvironment
 823
 824         return (board_dict, err_lines_summary, err_lines_boards,
 825                 warn_lines_summary, warn_lines_boards, config, environment)
 826
 827     def AddOutcome(self, board_dict, arch_list, changes, char, color):
 828         """Add an output to our list of outcomes for each architecture
 829
 830         This simple function adds failing boards (changes) to the
 831         relevant architecture string, so we can print the results out
 832         sorted by architecture.
 833
 834         Args:
 835              board_dict: Dict containing all boards
 836              arch_list: Dict keyed by arch name. Value is a string containing
 837                     a list of board names which failed for that arch.
 838              changes: List of boards to add to arch_list
 839              color: terminal.Colour object
 840         """
 841         done_arch = {}
 842         for target in changes:
 843             if target in board_dict:
 844                 arch = board_dict[target].arch
 845             else:
 846                 arch = 'unknown'
 847             str = self.col.Color(color, ' ' + target)
 848             if not arch in done_arch:
 849                 str = ' %s  %s' % (self.col.Color(color, char), str)
 850                 done_arch[arch] = True
 851             if not arch in arch_list:
 852                 arch_list[arch] = str
 853             else:
 854                 arch_list[arch] += str
 855
 856
 857     def ColourNum(self, num):
 858         color = self.col.RED if num > 0 else self.col.GREEN
 859         if num == 0:
 860             return '0'
 861         return self.col.Color(color, str(num))
 862
 863     def ResetResultSummary(self, board_selected):
 864         """Reset the results summary ready for use.
 865
 866         Set up the base board list to be all those selected, and set the
 867         error lines to empty.
 868
 869         Following this, calls to PrintResultSummary() will use this
 870         information to work out what has changed.
 871
 872         Args:
 873             board_selected: Dict containing boards to summarise, keyed by
 874                 board.target
 875         """
 876         self._base_board_dict = {}
 877         for board in board_selected:
 878             self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
 879                                                            {})
 880         self._base_err_lines = []
 881         self._base_warn_lines = []
 882         self._base_err_line_boards = {}
 883         self._base_warn_line_boards = {}
 884         self._base_config = None
 885         self._base_environment = None
 886
 887     def PrintFuncSizeDetail(self, fname, old, new):
 888         grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
 889         delta, common = [], {}
 890
 891         for a in old:
 892             if a in new:
 893                 common[a] = 1
 894
 895         for name in old:
 896             if name not in common:
 897                 remove += 1
 898                 down += old[name]
 899                 delta.append([-old[name], name])
 900
 901         for name in new:
 902             if name not in common:
 903                 add += 1
 904                 up += new[name]
 905                 delta.append([new[name], name])
 906
 907         for name in common:
 908                 diff = new.get(name, 0) - old.get(name, 0)
 909                 if diff > 0:
 910                     grow, up = grow + 1, up + diff
 911                 elif diff < 0:
 912                     shrink, down = shrink + 1, down - diff
 913                 delta.append([diff, name])
 914
 915         delta.sort()
 916         delta.reverse()
 917
 918         args = [add, -remove, grow, -shrink, up, -down, up - down]
 919         if max(args) == 0 and min(args) == 0:
 920             return
 921         args = [self.ColourNum(x) for x in args]
 922         indent = ' ' * 15
 923         Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
 924               tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
 925         Print('%s  %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
 926                                          'delta'))
 927         for diff, name in delta:
 928             if diff:
 929                 color = self.col.RED if diff > 0 else self.col.GREEN
 930                 msg = '%s  %-38s %7s %7s %+7d' % (indent, name,
 931                         old.get(name, '-'), new.get(name,'-'), diff)
 932                 Print(msg, colour=color)
 933
 934
 935     def PrintSizeDetail(self, target_list, show_bloat):
 936         """Show details size information for each board
 937
 938         Args:
 939             target_list: List of targets, each a dict containing:
 940                     'target': Target name
 941                     'total_diff': Total difference in bytes across all areas
 942                     <part_name>: Difference for that part
 943             show_bloat: Show detail for each function
 944         """
 945         targets_by_diff = sorted(target_list, reverse=True,
 946         key=lambda x: x['_total_diff'])
 947         for result in targets_by_diff:
 948             printed_target = False
 949             for name in sorted(result):
 950                 diff = result[name]
 951                 if name.startswith('_'):
 952                     continue
 953                 if diff != 0:
 954                     color = self.col.RED if diff > 0 else self.col.GREEN
 955                 msg = ' %s %+d' % (name, diff)
 956                 if not printed_target:
 957                     Print('%10s  %-15s:' % ('', result['_target']),
 958                           newline=False)
 959                     printed_target = True
 960                 Print(msg, colour=color, newline=False)
 961             if printed_target:
 962                 Print()
 963                 if show_bloat:
 964                     target = result['_target']
 965                     outcome = result['_outcome']
 966                     base_outcome = self._base_board_dict[target]
 967                     for fname in outcome.func_sizes:
 968                         self.PrintFuncSizeDetail(fname,
 969                                                  base_outcome.func_sizes[fname],
 970                                                  outcome.func_sizes[fname])
 971
 972
 973     def PrintSizeSummary(self, board_selected, board_dict, show_detail,
 974                          show_bloat):
 975         """Print a summary of image sizes broken down by section.
 976
 977         The summary takes the form of one line per architecture. The
 978         line contains deltas for each of the sections (+ means the section
 979         got bigger, - means smaller). The numbers are the average number
 980         of bytes that a board in this section increased by.
 981
 982         For example:
 983            powerpc: (622 boards)   text -0.0
 984           arm: (285 boards)   text -0.0
 985           nds32: (3 boards)   text -8.0
 986
 987         Args:
 988             board_selected: Dict containing boards to summarise, keyed by
 989                 board.target
 990             board_dict: Dict containing boards for which we built this
 991                 commit, keyed by board.target. The value is an Outcome object.
 992             show_detail: Show size delta detail for each board
 993             show_bloat: Show detail for each function
 994         """
 995         arch_list = {}
 996         arch_count = {}
 997
 998         # Calculate changes in size for different image parts
 999         # The previous sizes are in Board.sizes, for each board
1000         for target in board_dict:
1001             if target not in board_selected:
1002                 continue
1003             base_sizes = self._base_board_dict[target].sizes
1004             outcome = board_dict[target]
1005             sizes = outcome.sizes
1006
1007             # Loop through the list of images, creating a dict of size
1008             # changes for each image/part. We end up with something like
1009             # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1010             # which means that U-Boot data increased by 5 bytes and SPL
1011             # text decreased by 4.
1012             err = {'_target' : target}
1013             for image in sizes:
1014                 if image in base_sizes:
1015                     base_image = base_sizes[image]
1016                     # Loop through the text, data, bss parts
1017                     for part in sorted(sizes[image]):
1018                         diff = sizes[image][part] - base_image[part]
1019                         col = None
1020                         if diff:
1021                             if image == 'u-boot':
1022                                 name = part
1023                             else:
1024                                 name = image + ':' + part
1025                             err[name] = diff
1026             arch = board_selected[target].arch
1027             if not arch in arch_count:
1028                 arch_count[arch] = 1
1029             else:
1030                 arch_count[arch] += 1
1031             if not sizes:
1032                 pass    # Only add to our list when we have some stats
1033             elif not arch in arch_list:
1034                 arch_list[arch] = [err]
1035             else:
1036                 arch_list[arch].append(err)
1037
1038         # We now have a list of image size changes sorted by arch
1039         # Print out a summary of these
1040         for arch, target_list in arch_list.items():
1041             # Get total difference for each type
1042             totals = {}
1043             for result in target_list:
1044                 total = 0
1045                 for name, diff in result.items():
1046                     if name.startswith('_'):
1047                         continue
1048                     total += diff
1049                     if name in totals:
1050                         totals[name] += diff
1051                     else:
1052                         totals[name] = diff
1053                 result['_total_diff'] = total
1054                 result['_outcome'] = board_dict[result['_target']]
1055
1056             count = len(target_list)
1057             printed_arch = False
1058             for name in sorted(totals):
1059                 diff = totals[name]
1060                 if diff:
1061                     # Display the average difference in this name for this
1062                     # architecture
1063                     avg_diff = float(diff) / count
1064                     color = self.col.RED if avg_diff > 0 else self.col.GREEN
1065                     msg = ' %s %+1.1f' % (name, avg_diff)
1066                     if not printed_arch:
1067                         Print('%10s: (for %d/%d boards)' % (arch, count,
1068                               arch_count[arch]), newline=False)
1069                         printed_arch = True
1070                     Print(msg, colour=color, newline=False)
1071
1072             if printed_arch:
1073                 Print()
1074                 if show_detail:
1075                     self.PrintSizeDetail(target_list, show_bloat)
1076
1077
1078     def PrintResultSummary(self, board_selected, board_dict, err_lines,
1079                            err_line_boards, warn_lines, warn_line_boards,
1080                            config, environment, show_sizes, show_detail,
1081                            show_bloat, show_config, show_environment):
1082         """Compare results with the base results and display delta.
1083
1084         Only boards mentioned in board_selected will be considered. This
1085         function is intended to be called repeatedly with the results of
1086         each commit. It therefore shows a 'diff' between what it saw in
1087         the last call and what it sees now.
1088
1089         Args:
1090             board_selected: Dict containing boards to summarise, keyed by
1091                 board.target
1092             board_dict: Dict containing boards for which we built this
1093                 commit, keyed by board.target. The value is an Outcome object.
1094             err_lines: A list of errors for this commit, or [] if there is
1095                 none, or we don't want to print errors
1096             err_line_boards: Dict keyed by error line, containing a list of
1097                 the Board objects with that error
1098             warn_lines: A list of warnings for this commit, or [] if there is
1099                 none, or we don't want to print errors
1100             warn_line_boards: Dict keyed by warning line, containing a list of
1101                 the Board objects with that warning
1102             config: Dictionary keyed by filename - e.g. '.config'. Each
1103                     value is itself a dictionary:
1104                         key: config name
1105                         value: config value
1106             environment: Dictionary keyed by environment variable, Each
1107                      value is the value of environment variable.
1108             show_sizes: Show image size deltas
1109             show_detail: Show size delta detail for each board if show_sizes
1110             show_bloat: Show detail for each function
1111             show_config: Show config changes
1112             show_environment: Show environment changes
1113         """
1114         def _BoardList(line, line_boards):
1115             """Helper function to get a line of boards containing a line
1116
1117             Args:
1118                 line: Error line to search for
1119                 line_boards: boards to search, each a Board
1120             Return:
1121                 List of boards with that error line, or [] if the user has not
1122                     requested such a list
1123             """
1124             boards = []
1125             board_set = set()
1126             if self._list_error_boards:
1127                 for board in line_boards[line]:
1128                     if not board in board_set:
1129                         boards.append(board)
1130                         board_set.add(board)
1131             return boards
1132
1133         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1134                             char):
1135             """Calculate the required output based on changes in errors
1136
1137             Args:
1138                 base_lines: List of errors/warnings for previous commit
1139                 base_line_boards: Dict keyed by error line, containing a list
1140                     of the Board objects with that error in the previous commit
1141                 lines: List of errors/warning for this commit, each a str
1142                 line_boards: Dict keyed by error line, containing a list
1143                     of the Board objects with that error in this commit
1144                 char: Character representing error ('') or warning ('w'). The
1145                     broken ('+') or fixed ('-') characters are added in this
1146                     function
1147
1148             Returns:
1149                 Tuple
1150                     List of ErrLine objects for 'better' lines
1151                     List of ErrLine objects for 'worse' lines
1152             """
1153             better_lines = []
1154             worse_lines = []
1155             for line in lines:
1156                 if line not in base_lines:
1157                     errline = ErrLine(char + '+', _BoardList(line, line_boards),
1158                                       line)
1159                     worse_lines.append(errline)
1160             for line in base_lines:
1161                 if line not in lines:
1162                     errline = ErrLine(char + '-',
1163                                       _BoardList(line, base_line_boards), line)
1164                     better_lines.append(errline)
1165             return better_lines, worse_lines
1166
1167         def _CalcConfig(delta, name, config):
1168             """Calculate configuration changes
1169
1170             Args:
1171                 delta: Type of the delta, e.g. '+'
1172                 name: name of the file which changed (e.g. .config)
1173                 config: configuration change dictionary
1174                     key: config name
1175                     value: config value
1176             Returns:
1177                 String containing the configuration changes which can be
1178                     printed
1179             """
1180             out = ''
1181             for key in sorted(config.keys()):
1182                 out += '%s=%s ' % (key, config[key])
1183             return '%s %s: %s' % (delta, name, out)
1184
1185         def _AddConfig(lines, name, config_plus, config_minus, config_change):
1186             """Add changes in configuration to a list
1187
1188             Args:
1189                 lines: list to add to
1190                 name: config file name
1191                 config_plus: configurations added, dictionary
1192                     key: config name
1193                     value: config value
1194                 config_minus: configurations removed, dictionary
1195                     key: config name
1196                     value: config value
1197                 config_change: configurations changed, dictionary
1198                     key: config name
1199                     value: config value
1200             """
1201             if config_plus:
1202                 lines.append(_CalcConfig('+', name, config_plus))
1203             if config_minus:
1204                 lines.append(_CalcConfig('-', name, config_minus))
1205             if config_change:
1206                 lines.append(_CalcConfig('c', name, config_change))
1207
1208         def _OutputConfigInfo(lines):
1209             for line in lines:
1210                 if not line:
1211                     continue
1212                 if line[0] == '+':
1213                     col = self.col.GREEN
1214                 elif line[0] == '-':
1215                     col = self.col.RED
1216                 elif line[0] == 'c':
1217                     col = self.col.YELLOW
1218                 Print('   ' + line, newline=True, colour=col)
1219
1220         def _OutputErrLines(err_lines, colour):
1221             """Output the line of error/warning lines, if not empty
1222
1223             Also increments self._error_lines if err_lines not empty
1224
1225             Args:
1226                 err_lines: List of ErrLine objects, each an error or warning
1227                     line, possibly including a list of boards with that
1228                     error/warning
1229                 colour: Colour to use for output
1230             """
1231             if err_lines:
1232                 out_list = []
1233                 for line in err_lines:
1234                     boards = ''
1235                     names = [board.target for board in line.boards]
1236                     board_str = ' '.join(names) if names else ''
1237                     if board_str:
1238                         out = self.col.Color(colour, line.char + '(')
1239                         out += self.col.Color(self.col.MAGENTA, board_str,
1240                                               bright=False)
1241                         out += self.col.Color(colour, ') %s' % line.errline)
1242                     else:
1243                         out = self.col.Color(colour, line.char + line.errline)
1244                     out_list.append(out)
1245                 Print('\n'.join(out_list))
1246                 self._error_lines += 1
1247
1248
1249         ok_boards = []      # List of boards fixed since last commit
1250         warn_boards = []    # List of boards with warnings since last commit
1251         err_boards = []     # List of new broken boards since last commit
1252         new_boards = []     # List of boards that didn't exist last time
1253         unknown_boards = [] # List of boards that were not built
1254
1255         for target in board_dict:
1256             if target not in board_selected:
1257                 continue
1258
1259             # If the board was built last time, add its outcome to a list
1260             if target in self._base_board_dict:
1261                 base_outcome = self._base_board_dict[target].rc
1262                 outcome = board_dict[target]
1263                 if outcome.rc == OUTCOME_UNKNOWN:
1264                     unknown_boards.append(target)
1265                 elif outcome.rc < base_outcome:
1266                     if outcome.rc == OUTCOME_WARNING:
1267                         warn_boards.append(target)
1268                     else:
1269                         ok_boards.append(target)
1270                 elif outcome.rc > base_outcome:
1271                     if outcome.rc == OUTCOME_WARNING:
1272                         warn_boards.append(target)
1273                     else:
1274                         err_boards.append(target)
1275             else:
1276                 new_boards.append(target)
1277
1278         # Get a list of errors and warnings that have appeared, and disappeared
1279         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1280                 self._base_err_line_boards, err_lines, err_line_boards, '')
1281         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1282                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1283
1284         # Display results by arch
1285         if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1286                 worse_err, better_err, worse_warn, better_warn)):
1287             arch_list = {}
1288             self.AddOutcome(board_selected, arch_list, ok_boards, '',
1289                     self.col.GREEN)
1290             self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1291                     self.col.YELLOW)
1292             self.AddOutcome(board_selected, arch_list, err_boards, '+',
1293                     self.col.RED)
1294             self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1295             if self._show_unknown:
1296                 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1297                         self.col.MAGENTA)
1298             for arch, target_list in arch_list.items():
1299                 Print('%10s: %s' % (arch, target_list))
1300                 self._error_lines += 1
1301             _OutputErrLines(better_err, colour=self.col.GREEN)
1302             _OutputErrLines(worse_err, colour=self.col.RED)
1303             _OutputErrLines(better_warn, colour=self.col.CYAN)
1304             _OutputErrLines(worse_warn, colour=self.col.YELLOW)
1305
1306         if show_sizes:
1307             self.PrintSizeSummary(board_selected, board_dict, show_detail,
1308                                   show_bloat)
1309
1310         if show_environment and self._base_environment:
1311             lines = []
1312
1313             for target in board_dict:
1314                 if target not in board_selected:
1315                     continue
1316
1317                 tbase = self._base_environment[target]
1318                 tenvironment = environment[target]
1319                 environment_plus = {}
1320                 environment_minus = {}
1321                 environment_change = {}
1322                 base = tbase.environment
1323                 for key, value in tenvironment.environment.items():
1324                     if key not in base:
1325                         environment_plus[key] = value
1326                 for key, value in base.items():
1327                     if key not in tenvironment.environment:
1328                         environment_minus[key] = value
1329                 for key, value in base.items():
1330                     new_value = tenvironment.environment.get(key)
1331                     if new_value and value != new_value:
1332                         desc = '%s -> %s' % (value, new_value)
1333                         environment_change[key] = desc
1334
1335                 _AddConfig(lines, target, environment_plus, environment_minus,
1336                            environment_change)
1337
1338             _OutputConfigInfo(lines)
1339
1340         if show_config and self._base_config:
1341             summary = {}
1342             arch_config_plus = {}
1343             arch_config_minus = {}
1344             arch_config_change = {}
1345             arch_list = []
1346
1347             for target in board_dict:
1348                 if target not in board_selected:
1349                     continue
1350                 arch = board_selected[target].arch
1351                 if arch not in arch_list:
1352                     arch_list.append(arch)
1353
1354             for arch in arch_list:
1355                 arch_config_plus[arch] = {}
1356                 arch_config_minus[arch] = {}
1357                 arch_config_change[arch] = {}
1358                 for name in self.config_filenames:
1359                     arch_config_plus[arch][name] = {}
1360                     arch_config_minus[arch][name] = {}
1361                     arch_config_change[arch][name] = {}
1362
1363             for target in board_dict:
1364                 if target not in board_selected:
1365                     continue
1366
1367                 arch = board_selected[target].arch
1368
1369                 all_config_plus = {}
1370                 all_config_minus = {}
1371                 all_config_change = {}
1372                 tbase = self._base_config[target]
1373                 tconfig = config[target]
1374                 lines = []
1375                 for name in self.config_filenames:
1376                     if not tconfig.config[name]:
1377                         continue
1378                     config_plus = {}
1379                     config_minus = {}
1380                     config_change = {}
1381                     base = tbase.config[name]
1382                     for key, value in tconfig.config[name].items():
1383                         if key not in base:
1384                             config_plus[key] = value
1385                             all_config_plus[key] = value
1386                     for key, value in base.items():
1387                         if key not in tconfig.config[name]:
1388                             config_minus[key] = value
1389                             all_config_minus[key] = value
1390                     for key, value in base.items():
1391                         new_value = tconfig.config.get(key)
1392                         if new_value and value != new_value:
1393                             desc = '%s -> %s' % (value, new_value)
1394                             config_change[key] = desc
1395                             all_config_change[key] = desc
1396
1397                     arch_config_plus[arch][name].update(config_plus)
1398                     arch_config_minus[arch][name].update(config_minus)
1399                     arch_config_change[arch][name].update(config_change)
1400
1401                     _AddConfig(lines, name, config_plus, config_minus,
1402                                config_change)
1403                 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1404                            all_config_change)
1405                 summary[target] = '\n'.join(lines)
1406
1407             lines_by_target = {}
1408             for target, lines in summary.items():
1409                 if lines in lines_by_target:
1410                     lines_by_target[lines].append(target)
1411                 else:
1412                     lines_by_target[lines] = [target]
1413
1414             for arch in arch_list:
1415                 lines = []
1416                 all_plus = {}
1417                 all_minus = {}
1418                 all_change = {}
1419                 for name in self.config_filenames:
1420                     all_plus.update(arch_config_plus[arch][name])
1421                     all_minus.update(arch_config_minus[arch][name])
1422                     all_change.update(arch_config_change[arch][name])
1423                     _AddConfig(lines, name, arch_config_plus[arch][name],
1424                                arch_config_minus[arch][name],
1425                                arch_config_change[arch][name])
1426                 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1427                 #arch_summary[target] = '\n'.join(lines)
1428                 if lines:
1429                     Print('%s:' % arch)
1430                     _OutputConfigInfo(lines)
1431
1432             for lines, targets in lines_by_target.items():
1433                 if not lines:
1434                     continue
1435                 Print('%s :' % ' '.join(sorted(targets)))
1436                 _OutputConfigInfo(lines.split('\n'))
1437
1438
1439         # Save our updated information for the next call to this function
1440         self._base_board_dict = board_dict
1441         self._base_err_lines = err_lines
1442         self._base_warn_lines = warn_lines
1443         self._base_err_line_boards = err_line_boards
1444         self._base_warn_line_boards = warn_line_boards
1445         self._base_config = config
1446         self._base_environment = environment
1447
1448         # Get a list of boards that did not get built, if needed
1449         not_built = []
1450         for board in board_selected:
1451             if not board in board_dict:
1452                 not_built.append(board)
1453         if not_built:
1454             Print("Boards not built (%d): %s" % (len(not_built),
1455                   ', '.join(not_built)))
1456
1457     def ProduceResultSummary(self, commit_upto, commits, board_selected):
1458             (board_dict, err_lines, err_line_boards, warn_lines,
1459              warn_line_boards, config, environment) = self.GetResultSummary(
1460                     board_selected, commit_upto,
1461                     read_func_sizes=self._show_bloat,
1462                     read_config=self._show_config,
1463                     read_environment=self._show_environment)
1464             if commits:
1465                 msg = '%02d: %s' % (commit_upto + 1,
1466                         commits[commit_upto].subject)
1467                 Print(msg, colour=self.col.BLUE)
1468             self.PrintResultSummary(board_selected, board_dict,
1469                     err_lines if self._show_errors else [], err_line_boards,
1470                     warn_lines if self._show_errors else [], warn_line_boards,
1471                     config, environment, self._show_sizes, self._show_detail,
1472                     self._show_bloat, self._show_config, self._show_environment)
1473
1474     def ShowSummary(self, commits, board_selected):
1475         """Show a build summary for U-Boot for a given board list.
1476
1477         Reset the result summary, then repeatedly call GetResultSummary on
1478         each commit's results, then display the differences we see.
1479
1480         Args:
1481             commit: Commit objects to summarise
1482             board_selected: Dict containing boards to summarise
1483         """
1484         self.commit_count = len(commits) if commits else 1
1485         self.commits = commits
1486         self.ResetResultSummary(board_selected)
1487         self._error_lines = 0
1488
1489         for commit_upto in range(0, self.commit_count, self._step):
1490             self.ProduceResultSummary(commit_upto, commits, board_selected)
1491         if not self._error_lines:
1492             Print('(no errors to report)', colour=self.col.GREEN)
1493
1494
1495     def SetupBuild(self, board_selected, commits):
1496         """Set up ready to start a build.
1497
1498         Args:
1499             board_selected: Selected boards to build
1500             commits: Selected commits to build
1501         """
1502         # First work out how many commits we will build
1503         count = (self.commit_count + self._step - 1) // self._step
1504         self.count = len(board_selected) * count
1505         self.upto = self.warned = self.fail = 0
1506         self._timestamps = collections.deque()
1507
1508     def GetThreadDir(self, thread_num):
1509         """Get the directory path to the working dir for a thread.
1510
1511         Args:
1512             thread_num: Number of thread to check.
1513         """
1514         if self.work_in_output:
1515             return self._working_dir
1516         return os.path.join(self._working_dir, '%02d' % thread_num)
1517
1518     def _PrepareThread(self, thread_num, setup_git):
1519         """Prepare the working directory for a thread.
1520
1521         This clones or fetches the repo into the thread's work directory.
1522
1523         Args:
1524             thread_num: Thread number (0, 1, ...)
1525             setup_git: True to set up a git repo clone
1526         """
1527         thread_dir = self.GetThreadDir(thread_num)
1528         builderthread.Mkdir(thread_dir)
1529         git_dir = os.path.join(thread_dir, '.git')
1530
1531         # Clone the repo if it doesn't already exist
1532         # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1533         # we have a private index but uses the origin repo's contents?
1534         if setup_git and self.git_dir:
1535             src_dir = os.path.abspath(self.git_dir)
1536             if os.path.exists(git_dir):
1537                 Print('\rFetching repo for thread %d' % thread_num,
1538                       newline=False)
1539                 gitutil.Fetch(git_dir, thread_dir)
1540                 terminal.PrintClear()
1541             else:
1542                 Print('\rCloning repo for thread %d' % thread_num,
1543                       newline=False)
1544                 gitutil.Clone(src_dir, thread_dir)
1545                 terminal.PrintClear()
1546
1547     def _PrepareWorkingSpace(self, max_threads, setup_git):
1548         """Prepare the working directory for use.
1549
1550         Set up the git repo for each thread.
1551
1552         Args:
1553             max_threads: Maximum number of threads we expect to need.
1554             setup_git: True to set up a git repo clone
1555         """
1556         builderthread.Mkdir(self._working_dir)
1557         for thread in range(max_threads):
1558             self._PrepareThread(thread, setup_git)
1559
1560     def _GetOutputSpaceRemovals(self):
1561         """Get the output directories ready to receive files.
1562
1563         Figure out what needs to be deleted in the output directory before it
1564         can be used. We only delete old buildman directories which have the
1565         expected name pattern. See _GetOutputDir().
1566
1567         Returns:
1568             List of full paths of directories to remove
1569         """
1570         if not self.commits:
1571             return
1572         dir_list = []
1573         for commit_upto in range(self.commit_count):
1574             dir_list.append(self._GetOutputDir(commit_upto))
1575
1576         to_remove = []
1577         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1578             if dirname not in dir_list:
1579                 leaf = dirname[len(self.base_dir) + 1:]
1580                 m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1581                 if m:
1582                     to_remove.append(dirname)
1583         return to_remove
1584
1585     def _PrepareOutputSpace(self):
1586         """Get the output directories ready to receive files.
1587
1588         We delete any output directories which look like ones we need to
1589         create. Having left over directories is confusing when the user wants
1590         to check the output manually.
1591         """
1592         to_remove = self._GetOutputSpaceRemovals()
1593         if to_remove:
1594             Print('Removing %d old build directories...' % len(to_remove),
1595                   newline=False)
1596             for dirname in to_remove:
1597                 shutil.rmtree(dirname)
1598             terminal.PrintClear()
1599
1600     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1601         """Build all commits for a list of boards
1602
1603         Args:
1604             commits: List of commits to be build, each a Commit object
1605             boards_selected: Dict of selected boards, key is target name,
1606                     value is Board object
1607             keep_outputs: True to save build output files
1608             verbose: Display build results as they are completed
1609         Returns:
1610             Tuple containing:
1611                 - number of boards that failed to build
1612                 - number of boards that issued warnings
1613         """
1614         self.commit_count = len(commits) if commits else 1
1615         self.commits = commits
1616         self._verbose = verbose
1617
1618         self.ResetResultSummary(board_selected)
1619         builderthread.Mkdir(self.base_dir, parents = True)
1620         self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1621                 commits is not None)
1622         self._PrepareOutputSpace()
1623         Print('\rStarting build...', newline=False)
1624         self.SetupBuild(board_selected, commits)
1625         self.ProcessResult(None)
1626
1627         # Create jobs to build all commits for each board
1628         for brd in board_selected.values():
1629             job = builderthread.BuilderJob()
1630             job.board = brd
1631             job.commits = commits
1632             job.keep_outputs = keep_outputs
1633             job.work_in_output = self.work_in_output
1634             job.step = self._step
1635             self.queue.put(job)
1636
1637         term = threading.Thread(target=self.queue.join)
1638         term.setDaemon(True)
1639         term.start()
1640         while term.isAlive():
1641             term.join(100)
1642
1643         # Wait until we have processed all output
1644         self.out_queue.join()
1645         Print()
1646
1647         msg = 'Completed: %d total built' % self.count
1648         if self.already_done:
1649            msg += ' (%d previously' % self.already_done
1650            if self.already_done != self.count:
1651                msg += ', %d newly' % (self.count - self.already_done)
1652            msg += ')'
1653         duration = datetime.now() - self._start_time
1654         if duration > timedelta(microseconds=1000000):
1655             if duration.microseconds >= 500000:
1656                 duration = duration + timedelta(seconds=1)
1657             duration = duration - timedelta(microseconds=duration.microseconds)
1658             msg += ', duration %s' % duration
1659         Print(msg)
1660
1661         return (self.fail, self.warned)