Merge https://gitlab.denx.de/u-boot/custodians/u-boot-spi into next
[platform/kernel/u-boot.git] / tools / buildman / builder.py
index 05f8299..f2756ea 100644 (file)
@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
 import glob
 import os
 import re
-import Queue
+import queue
 import shutil
 import signal
 import string
@@ -17,13 +17,12 @@ import sys
 import threading
 import time
 
-import builderthread
-import command
-import gitutil
-import terminal
-from terminal import Print
-import toolchain
-
+from buildman import builderthread
+from buildman import toolchain
+from patman import command
+from patman import gitutil
+from patman import terminal
+from patman.terminal import Print
 
 """
 Theory of Operation
@@ -71,12 +70,12 @@ As an example, say we are building branch 'us-net' for boards 'sandbox' and
 like this:
 
 us-net/             base directory
-    01_of_02_g4ed4ebc_net--Add-tftp-speed-/
+    01_g4ed4ebc_net--Add-tftp-speed-/
         sandbox/
             u-boot.bin
         seaboard/
             u-boot.bin
-    02_of_02_g4ed4ebc_net--Check-tftp-comp/
+    02_g4ed4ebc_net--Check-tftp-comp/
         sandbox/
             u-boot.bin
         seaboard/
@@ -91,12 +90,20 @@ u-boot/             source directory
     .git/           repository
 """
 
+"""Holds information about a particular error line we are outputing
+
+   char: Character representation: '+': error, '-': fixed error, 'w+': warning,
+       'w-' = fixed warning
+   boards: List of Board objects which have line in the error/warning output
+   errline: The text of the error line
+"""
+ErrLine = collections.namedtuple('ErrLine', 'char,boards,errline')
+
 # Possible build outcomes
-OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
+OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
 
 # Translate a commit subject into a valid filename (and handle unicode)
-trans_valid_chars = string.maketrans('/: ', '---')
-trans_valid_chars = trans_valid_chars.decode('latin-1')
+trans_valid_chars = str.maketrans('/: ', '---')
 
 BASE_CONFIG_FILENAMES = [
     'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
@@ -122,8 +129,8 @@ class Config:
     def __hash__(self):
         val = 0
         for fname in self.config:
-            for key, value in self.config[fname].iteritems():
-                print key, value
+            for key, value in self.config[fname].items():
+                print(key, value)
                 val = val ^ hash(key) & hash(value)
         return val
 
@@ -155,8 +162,6 @@ class Builder:
         force_build_failures: If a previously-built build (i.e. built on
             a previous run of buildman) is marked as failed, rebuild it.
         git_dir: Git directory containing source repository
-        last_line_len: Length of the last line we printed (used for erasing
-            it with new progress information)
         num_jobs: Number of jobs to run at once (passed to make as -j)
         num_threads: Number of builder threads to run
         out_queue: Queue of results to process
@@ -175,6 +180,8 @@ class Builder:
         in_tree: Build U-Boot in-tree instead of specifying an output
             directory separate from the source code. This option is really
             only useful for testing in-tree builds.
+        work_in_output: Use the output directory as the work directory and
+            don't write to a separate output directory.
 
     Private members:
         _base_board_dict: Last-summarised Dict of boards
@@ -185,6 +192,7 @@ class Builder:
         _next_delay_update: Next time we plan to display a progress update
                 (datatime)
         _show_unknown: Show unknown boards (those not built) in summary
+        _start_time: Start time for the build
         _timestamps: List of timestamps for the completion of the last
             last _timestamp_count builds. Each is a datetime object.
         _timestamp_count: Number of timestamps to keep in our list.
@@ -223,9 +231,9 @@ class Builder:
     def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
                  gnu_make='make', checkout=True, show_unknown=True, step=1,
                  no_subdirs=False, full_path=False, verbose_build=False,
-                 incremental=False, per_board_out_dir=False,
+                 mrproper=False, per_board_out_dir=False,
                  config_only=False, squash_config_y=False,
-                 warnings_as_errors=False):
+                 warnings_as_errors=False, work_in_output=False):
         """Create a new Builder object
 
         Args:
@@ -244,17 +252,21 @@ class Builder:
             full_path: Return the full path in CROSS_COMPILE and don't set
                 PATH
             verbose_build: Run build with V=1 and don't use 'make -s'
-            incremental: Always perform incremental builds; don't run make
-                mrproper when configuring
+            mrproper: Always run 'make mrproper' when configuring
             per_board_out_dir: Build in a separate persistent directory per
                 board rather than a thread-specific directory
             config_only: Only configure each build, don't build it
             squash_config_y: Convert CONFIG options with the value 'y' to '1'
             warnings_as_errors: Treat all compiler warnings as errors
+            work_in_output: Use the output directory as the work directory and
+                don't write to a separate output directory.
         """
         self.toolchains = toolchains
         self.base_dir = base_dir
-        self._working_dir = os.path.join(base_dir, '.bm-work')
+        if work_in_output:
+            self._working_dir = base_dir
+        else:
+            self._working_dir = os.path.join(base_dir, '.bm-work')
         self.threads = []
         self.do_make = self.Make
         self.gnu_make = gnu_make
@@ -269,6 +281,7 @@ class Builder:
         self._build_period_us = None
         self._complete_delay = None
         self._next_delay_update = datetime.now()
+        self._start_time = datetime.now()
         self.force_config_on_failure = True
         self.force_build_failures = False
         self.force_reconfig = False
@@ -281,6 +294,7 @@ class Builder:
         self.config_only = config_only
         self.squash_config_y = squash_config_y
         self.config_filenames = BASE_CONFIG_FILENAMES
+        self.work_in_output = work_in_output
         if not self.squash_config_y:
             self.config_filenames += EXTRA_CONFIG_FILENAMES
 
@@ -290,18 +304,20 @@ class Builder:
         self._re_function = re.compile('(.*): In function.*')
         self._re_files = re.compile('In file included from.*')
         self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
+        self._re_dtb_warning = re.compile('(.*): Warning .*')
         self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
+        self._re_migration_warning = re.compile(r'^={21} WARNING ={22}\n.*\n=+\n',
+                                                re.MULTILINE | re.DOTALL)
 
-        self.queue = Queue.Queue()
-        self.out_queue = Queue.Queue()
+        self.queue = queue.Queue()
+        self.out_queue = queue.Queue()
         for i in range(self.num_threads):
-            t = builderthread.BuilderThread(self, i, incremental,
+            t = builderthread.BuilderThread(self, i, mrproper,
                     per_board_out_dir)
             t.setDaemon(True)
             t.start()
             self.threads.append(t)
 
-        self.last_line_len = 0
         t = builderthread.ResultThread(self)
         t.setDaemon(True)
         t.start()
@@ -324,16 +340,22 @@ class Builder:
     def SetDisplayOptions(self, show_errors=False, show_sizes=False,
                           show_detail=False, show_bloat=False,
                           list_error_boards=False, show_config=False,
-                          show_environment=False):
+                          show_environment=False, filter_dtb_warnings=False,
+                          filter_migration_warnings=False):
         """Setup display options for the builder.
 
-        show_errors: True to show summarised error/warning info
-        show_sizes: Show size deltas
-        show_detail: Show detail for each board
-        show_bloat: Show detail for each function
-        list_error_boards: Show the boards which caused each error/warning
-        show_config: Show config deltas
-        show_environment: Show environment deltas
+        Args:
+            show_errors: True to show summarised error/warning info
+            show_sizes: Show size deltas
+            show_detail: Show size delta detail for each board if show_sizes
+            show_bloat: Show detail for each function
+            list_error_boards: Show the boards which caused each error/warning
+            show_config: Show config deltas
+            show_environment: Show environment deltas
+            filter_dtb_warnings: Filter out any warnings from the device-tree
+                compiler
+            filter_migration_warnings: Filter out any warnings about migrating
+                a board to driver model
         """
         self._show_errors = show_errors
         self._show_sizes = show_sizes
@@ -342,6 +364,8 @@ class Builder:
         self._list_error_boards = list_error_boards
         self._show_config = show_config
         self._show_environment = show_environment
+        self._filter_dtb_warnings = filter_dtb_warnings
+        self._filter_migration_warnings = filter_migration_warnings
 
     def _AddTimestamp(self):
         """Add a new timestamp to the list and record the build period.
@@ -372,22 +396,6 @@ class Builder:
             self._timestamps.popleft()
             count -= 1
 
-    def ClearLine(self, length):
-        """Clear any characters on the current line
-
-        Make way for a new line of length 'length', by outputting enough
-        spaces to clear out the old line. Then remember the new length for
-        next time.
-
-        Args:
-            length: Length of new line, in characters
-        """
-        if length < self.last_line_len:
-            Print(' ' * (self.last_line_len - length), newline=False)
-            Print('\r', newline=False)
-        self.last_line_len = length
-        sys.stdout.flush()
-
     def SelectCommit(self, commit, checkout=True):
         """Checkout the selected commit for this build
         """
@@ -433,8 +441,7 @@ class Builder:
             if result.already_done:
                 self.already_done += 1
             if self._verbose:
-                Print('\r', newline=False)
-                self.ClearLine(0)
+                terminal.PrintClear()
                 boards_selected = {target : result.brd}
                 self.ResetResultSummary(boards_selected)
                 self.ProduceResultSummary(result.commit_upto, self.commits,
@@ -448,22 +455,21 @@ class Builder:
         line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
         line += self.col.Color(self.col.RED, '%5d' % self.fail)
 
-        name = ' /%-5d  ' % self.count
+        line += ' /%-5d  ' % self.count
+        remaining = self.count - self.upto
+        if remaining:
+            line += self.col.Color(self.col.MAGENTA, ' -%-5d  ' % remaining)
+        else:
+            line += ' ' * 8
 
         # Add our current completion time estimate
         self._AddTimestamp()
         if self._complete_delay:
-            name += '%s  : ' % self._complete_delay
-        # When building all boards for a commit, we can print a commit
-        # progress message.
-        if result and result.commit_upto is None:
-            name += 'commit %2d/%-3d' % (self.commit_upto + 1,
-                    self.commit_count)
-
-        name += target
-        Print(line + name, newline=False)
-        length = 16 + len(name)
-        self.ClearLine(length)
+            line += '%s  : ' % self._complete_delay
+
+        line += target
+        terminal.PrintClear()
+        Print(line, newline=False, limit_to_line=True)
 
     def _GetOutputDir(self, commit_upto):
         """Get the name of the output directory for a commit number
@@ -473,12 +479,16 @@ class Builder:
         Args:
             commit_upto: Commit number to use (0..self.count-1)
         """
+        if self.work_in_output:
+            return self._working_dir
+
         commit_dir = None
         if self.commits:
             commit = self.commits[commit_upto]
             subject = commit.subject.translate(trans_valid_chars)
-            commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
-                    self.commit_count, commit.hash, subject[:20]))
+            # See _GetOutputSpaceRemovals() which parses this name
+            commit_dir = ('%02d_g%s_%s' % (commit_upto + 1,
+                    commit.hash, subject[:20]))
         elif not self.no_subdirs:
             commit_dir = 'current'
         if not commit_dir:
@@ -495,6 +505,8 @@ class Builder:
             target: Target name
         """
         output_dir = self._GetOutputDir(commit_upto)
+        if self.work_in_output:
+            return output_dir
         return os.path.join(output_dir, target)
 
     def GetDoneFile(self, commit_upto, target):
@@ -558,9 +570,16 @@ class Builder:
             New list with only interesting lines included
         """
         out_lines = []
+        if self._filter_migration_warnings:
+            text = '\n'.join(lines)
+            text = self._re_migration_warning.sub('', text)
+            lines = text.splitlines()
         for line in lines:
-            if not self.re_make_err.search(line):
-                out_lines.append(line)
+            if self.re_make_err.search(line):
+                continue
+            if self._filter_dtb_warnings and self._re_dtb_warning.search(line):
+                continue
+            out_lines.append(line)
         return out_lines
 
     def ReadFuncSizes(self, fname, fd):
@@ -577,7 +596,8 @@ class Builder:
         sym = {}
         for line in fd.readlines():
             try:
-                size, type, name = line[:-1].split()
+                if line.strip():
+                    size, type, name = line[:-1].split()
             except:
                 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
                 continue
@@ -672,7 +692,12 @@ class Builder:
         environment = {}
         if os.path.exists(done_file):
             with open(done_file, 'r') as fd:
-                return_code = int(fd.readline())
+                try:
+                    return_code = int(fd.readline())
+                except ValueError:
+                    # The file may be empty due to running out of disk space.
+                    # Try a rebuild
+                    return_code = 1
                 err_lines = []
                 err_file = self.GetErrFile(commit_upto, target)
                 if os.path.exists(err_file):
@@ -775,7 +800,7 @@ class Builder:
         config = {}
         environment = {}
 
-        for board in boards_selected.itervalues():
+        for board in boards_selected.values():
             outcome = self.GetBuildOutcome(commit_upto, board.target,
                                            read_func_sizes, read_config,
                                            read_environment)
@@ -788,7 +813,8 @@ class Builder:
                             self._re_files.match(line)):
                         last_func = line
                     else:
-                        is_warning = self._re_warning.match(line)
+                        is_warning = (self._re_warning.match(line) or
+                                      self._re_dtb_warning.match(line))
                         is_note = self._re_note.match(line)
                         if is_warning or (last_was_warning and is_note):
                             if last_func:
@@ -807,13 +833,13 @@ class Builder:
             tconfig = Config(self.config_filenames, board.target)
             for fname in self.config_filenames:
                 if outcome.config:
-                    for key, value in outcome.config[fname].iteritems():
+                    for key, value in outcome.config[fname].items():
                         tconfig.Add(fname, key, value)
             config[board.target] = tconfig
 
             tenvironment = Environment(board.target)
             if outcome.environment:
-                for key, value in outcome.environment.iteritems():
+                for key, value in outcome.environment.items():
                     tenvironment.Add(key, value)
             environment[board.target] = tenvironment
 
@@ -972,7 +998,7 @@ class Builder:
 
         The summary takes the form of one line per architecture. The
         line contains deltas for each of the sections (+ means the section
-        got bigger, - means smaller). The nunmbers are the average number
+        got bigger, - means smaller). The numbers are the average number
         of bytes that a board in this section increased by.
 
         For example:
@@ -985,7 +1011,7 @@ class Builder:
                 board.target
             board_dict: Dict containing boards for which we built this
                 commit, keyed by board.target. The value is an Outcome object.
-            show_detail: Show detail for each board
+            show_detail: Show size delta detail for each board
             show_bloat: Show detail for each function
         """
         arch_list = {}
@@ -1033,12 +1059,12 @@ class Builder:
 
         # We now have a list of image size changes sorted by arch
         # Print out a summary of these
-        for arch, target_list in arch_list.iteritems():
+        for arch, target_list in arch_list.items():
             # Get total difference for each type
             totals = {}
             for result in target_list:
                 total = 0
-                for name, diff in result.iteritems():
+                for name, diff in result.items():
                     if name.startswith('_'):
                         continue
                     total += diff
@@ -1102,7 +1128,7 @@ class Builder:
             environment: Dictionary keyed by environment variable, Each
                      value is the value of environment variable.
             show_sizes: Show image size deltas
-            show_detail: Show detail for each board
+            show_detail: Show size delta detail for each board if show_sizes
             show_bloat: Show detail for each function
             show_config: Show config changes
             show_environment: Show environment changes
@@ -1112,32 +1138,52 @@ class Builder:
 
             Args:
                 line: Error line to search for
+                line_boards: boards to search, each a Board
             Return:
-                String containing a list of boards with that error line, or
-                '' if the user has not requested such a list
+                List of boards with that error line, or [] if the user has not
+                    requested such a list
             """
+            boards = []
+            board_set = set()
             if self._list_error_boards:
-                names = []
                 for board in line_boards[line]:
-                    if not board.target in names:
-                        names.append(board.target)
-                names_str = '(%s) ' % ','.join(names)
-            else:
-                names_str = ''
-            return names_str
+                    if not board in board_set:
+                        boards.append(board)
+                        board_set.add(board)
+            return boards
 
         def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
                             char):
+            """Calculate the required output based on changes in errors
+
+            Args:
+                base_lines: List of errors/warnings for previous commit
+                base_line_boards: Dict keyed by error line, containing a list
+                    of the Board objects with that error in the previous commit
+                lines: List of errors/warning for this commit, each a str
+                line_boards: Dict keyed by error line, containing a list
+                    of the Board objects with that error in this commit
+                char: Character representing error ('') or warning ('w'). The
+                    broken ('+') or fixed ('-') characters are added in this
+                    function
+
+            Returns:
+                Tuple
+                    List of ErrLine objects for 'better' lines
+                    List of ErrLine objects for 'worse' lines
+            """
             better_lines = []
             worse_lines = []
             for line in lines:
                 if line not in base_lines:
-                    worse_lines.append(char + '+' +
-                            _BoardList(line, line_boards) + line)
+                    errline = ErrLine(char + '+', _BoardList(line, line_boards),
+                                      line)
+                    worse_lines.append(errline)
             for line in base_lines:
                 if line not in lines:
-                    better_lines.append(char + '-' +
-                            _BoardList(line, base_line_boards) + line)
+                    errline = ErrLine(char + '-',
+                                      _BoardList(line, base_line_boards), line)
+                    better_lines.append(errline)
             return better_lines, worse_lines
 
         def _CalcConfig(delta, name, config):
@@ -1193,11 +1239,40 @@ class Builder:
                     col = self.col.YELLOW
                 Print('   ' + line, newline=True, colour=col)
 
+        def _OutputErrLines(err_lines, colour):
+            """Output the line of error/warning lines, if not empty
 
-        better = []     # List of boards fixed since last commit
-        worse = []      # List of new broken boards since last commit
-        new = []        # List of boards that didn't exist last time
-        unknown = []    # List of boards that were not built
+            Also increments self._error_lines if err_lines not empty
+
+            Args:
+                err_lines: List of ErrLine objects, each an error or warning
+                    line, possibly including a list of boards with that
+                    error/warning
+                colour: Colour to use for output
+            """
+            if err_lines:
+                out_list = []
+                for line in err_lines:
+                    boards = ''
+                    names = [board.target for board in line.boards]
+                    board_str = ' '.join(names) if names else ''
+                    if board_str:
+                        out = self.col.Color(colour, line.char + '(')
+                        out += self.col.Color(self.col.MAGENTA, board_str,
+                                              bright=False)
+                        out += self.col.Color(colour, ') %s' % line.errline)
+                    else:
+                        out = self.col.Color(colour, line.char + line.errline)
+                    out_list.append(out)
+                Print('\n'.join(out_list))
+                self._error_lines += 1
+
+
+        ok_boards = []      # List of boards fixed since last commit
+        warn_boards = []    # List of boards with warnings since last commit
+        err_boards = []     # List of new broken boards since last commit
+        new_boards = []     # List of boards that didn't exist last time
+        unknown_boards = [] # List of boards that were not built
 
         for target in board_dict:
             if target not in board_selected:
@@ -1208,47 +1283,47 @@ class Builder:
                 base_outcome = self._base_board_dict[target].rc
                 outcome = board_dict[target]
                 if outcome.rc == OUTCOME_UNKNOWN:
-                    unknown.append(target)
+                    unknown_boards.append(target)
                 elif outcome.rc < base_outcome:
-                    better.append(target)
+                    if outcome.rc == OUTCOME_WARNING:
+                        warn_boards.append(target)
+                    else:
+                        ok_boards.append(target)
                 elif outcome.rc > base_outcome:
-                    worse.append(target)
+                    if outcome.rc == OUTCOME_WARNING:
+                        warn_boards.append(target)
+                    else:
+                        err_boards.append(target)
             else:
-                new.append(target)
+                new_boards.append(target)
 
-        # Get a list of errors that have appeared, and disappeared
+        # Get a list of errors and warnings that have appeared, and disappeared
         better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
                 self._base_err_line_boards, err_lines, err_line_boards, '')
         better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
                 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
 
         # Display results by arch
-        if (better or worse or unknown or new or worse_err or better_err
-                or worse_warn or better_warn):
+        if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
+                worse_err, better_err, worse_warn, better_warn)):
             arch_list = {}
-            self.AddOutcome(board_selected, arch_list, better, '',
+            self.AddOutcome(board_selected, arch_list, ok_boards, '',
                     self.col.GREEN)
-            self.AddOutcome(board_selected, arch_list, worse, '+',
+            self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
+                    self.col.YELLOW)
+            self.AddOutcome(board_selected, arch_list, err_boards, '+',
                     self.col.RED)
-            self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
+            self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
             if self._show_unknown:
-                self.AddOutcome(board_selected, arch_list, unknown, '?',
+                self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
                         self.col.MAGENTA)
-            for arch, target_list in arch_list.iteritems():
+            for arch, target_list in arch_list.items():
                 Print('%10s: %s' % (arch, target_list))
                 self._error_lines += 1
-            if better_err:
-                Print('\n'.join(better_err), colour=self.col.GREEN)
-                self._error_lines += 1
-            if worse_err:
-                Print('\n'.join(worse_err), colour=self.col.RED)
-                self._error_lines += 1
-            if better_warn:
-                Print('\n'.join(better_warn), colour=self.col.CYAN)
-                self._error_lines += 1
-            if worse_warn:
-                Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
-                self._error_lines += 1
+            _OutputErrLines(better_err, colour=self.col.GREEN)
+            _OutputErrLines(worse_err, colour=self.col.RED)
+            _OutputErrLines(better_warn, colour=self.col.CYAN)
+            _OutputErrLines(worse_warn, colour=self.col.YELLOW)
 
         if show_sizes:
             self.PrintSizeSummary(board_selected, board_dict, show_detail,
@@ -1267,13 +1342,13 @@ class Builder:
                 environment_minus = {}
                 environment_change = {}
                 base = tbase.environment
-                for key, value in tenvironment.environment.iteritems():
+                for key, value in tenvironment.environment.items():
                     if key not in base:
                         environment_plus[key] = value
-                for key, value in base.iteritems():
+                for key, value in base.items():
                     if key not in tenvironment.environment:
                         environment_minus[key] = value
-                for key, value in base.iteritems():
+                for key, value in base.items():
                     new_value = tenvironment.environment.get(key)
                     if new_value and value != new_value:
                         desc = '%s -> %s' % (value, new_value)
@@ -1326,15 +1401,15 @@ class Builder:
                     config_minus = {}
                     config_change = {}
                     base = tbase.config[name]
-                    for key, value in tconfig.config[name].iteritems():
+                    for key, value in tconfig.config[name].items():
                         if key not in base:
                             config_plus[key] = value
                             all_config_plus[key] = value
-                    for key, value in base.iteritems():
+                    for key, value in base.items():
                         if key not in tconfig.config[name]:
                             config_minus[key] = value
                             all_config_minus[key] = value
-                    for key, value in base.iteritems():
+                    for key, value in base.items():
                         new_value = tconfig.config.get(key)
                         if new_value and value != new_value:
                             desc = '%s -> %s' % (value, new_value)
@@ -1352,7 +1427,7 @@ class Builder:
                 summary[target] = '\n'.join(lines)
 
             lines_by_target = {}
-            for target, lines in summary.iteritems():
+            for target, lines in summary.items():
                 if lines in lines_by_target:
                     lines_by_target[lines].append(target)
                 else:
@@ -1376,7 +1451,7 @@ class Builder:
                     Print('%s:' % arch)
                     _OutputConfigInfo(lines)
 
-            for lines, targets in lines_by_target.iteritems():
+            for lines, targets in lines_by_target.items():
                 if not lines:
                     continue
                 Print('%s :' % ' '.join(sorted(targets)))
@@ -1447,7 +1522,7 @@ class Builder:
             commits: Selected commits to build
         """
         # First work out how many commits we will build
-        count = (self.commit_count + self._step - 1) / self._step
+        count = (self.commit_count + self._step - 1) // self._step
         self.count = len(board_selected) * count
         self.upto = self.warned = self.fail = 0
         self._timestamps = collections.deque()
@@ -1458,6 +1533,8 @@ class Builder:
         Args:
             thread_num: Number of thread to check.
         """
+        if self.work_in_output:
+            return self._working_dir
         return os.path.join(self._working_dir, '%02d' % thread_num)
 
     def _PrepareThread(self, thread_num, setup_git):
@@ -1479,12 +1556,15 @@ class Builder:
         if setup_git and self.git_dir:
             src_dir = os.path.abspath(self.git_dir)
             if os.path.exists(git_dir):
+                Print('\rFetching repo for thread %d' % thread_num,
+                      newline=False)
                 gitutil.Fetch(git_dir, thread_dir)
+                terminal.PrintClear()
             else:
                 Print('\rCloning repo for thread %d' % thread_num,
                       newline=False)
                 gitutil.Clone(src_dir, thread_dir)
-                Print('\r%s\r' % (' ' * 30), newline=False)
+                terminal.PrintClear()
 
     def _PrepareWorkingSpace(self, max_threads, setup_git):
         """Prepare the working directory for use.
@@ -1499,12 +1579,15 @@ class Builder:
         for thread in range(max_threads):
             self._PrepareThread(thread, setup_git)
 
-    def _PrepareOutputSpace(self):
+    def _GetOutputSpaceRemovals(self):
         """Get the output directories ready to receive files.
 
-        We delete any output directories which look like ones we need to
-        create. Having left over directories is confusing when the user wants
-        to check the output manually.
+        Figure out what needs to be deleted in the output directory before it
+        can be used. We only delete old buildman directories which have the
+        expected name pattern. See _GetOutputDir().
+
+        Returns:
+            List of full paths of directories to remove
         """
         if not self.commits:
             return
@@ -1515,12 +1598,26 @@ class Builder:
         to_remove = []
         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
             if dirname not in dir_list:
-                to_remove.append(dirname)
+                leaf = dirname[len(self.base_dir) + 1:]
+                m =  re.match('[0-9]+_g[0-9a-f]+_.*', leaf)
+                if m:
+                    to_remove.append(dirname)
+        return to_remove
+
+    def _PrepareOutputSpace(self):
+        """Get the output directories ready to receive files.
+
+        We delete any output directories which look like ones we need to
+        create. Having left over directories is confusing when the user wants
+        to check the output manually.
+        """
+        to_remove = self._GetOutputSpaceRemovals()
         if to_remove:
-            Print('Removing %d old build directories' % len(to_remove),
+            Print('Removing %d old build directories...' % len(to_remove),
                   newline=False)
             for dirname in to_remove:
                 shutil.rmtree(dirname)
+            terminal.PrintClear()
 
     def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
         """Build all commits for a list of boards
@@ -1550,11 +1647,12 @@ class Builder:
         self.ProcessResult(None)
 
         # Create jobs to build all commits for each board
-        for brd in board_selected.itervalues():
+        for brd in board_selected.values():
             job = builderthread.BuilderJob()
             job.board = brd
             job.commits = commits
             job.keep_outputs = keep_outputs
+            job.work_in_output = self.work_in_output
             job.step = self._step
             self.queue.put(job)
 
@@ -1567,5 +1665,19 @@ class Builder:
         # Wait until we have processed all output
         self.out_queue.join()
         Print()
-        self.ClearLine(0)
+
+        msg = 'Completed: %d total built' % self.count
+        if self.already_done:
+           msg += ' (%d previously' % self.already_done
+           if self.already_done != self.count:
+               msg += ', %d newly' % (self.count - self.already_done)
+           msg += ')'
+        duration = datetime.now() - self._start_time
+        if duration > timedelta(microseconds=1000000):
+            if duration.microseconds >= 500000:
+                duration = duration + timedelta(seconds=1)
+            duration = duration - timedelta(microseconds=duration.microseconds)
+            msg += ', duration %s' % duration
+        Print(msg)
+
         return (self.fail, self.warned)