1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2013 The Chromium OS Authors.
4 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
8 from datetime import datetime, timedelta
24 from terminal import Print
31 Please see README for user documentation, and you should be familiar with
32 that before trying to make sense of this.
34 Buildman works by keeping the machine as busy as possible, building different
35 commits for different boards on multiple CPUs at once.
37 The source repo (self.git_dir) contains all the commits to be built. Each
38 thread works on a single board at a time. It checks out the first commit,
39 configures it for that board, then builds it. Then it checks out the next
40 commit and builds it (typically without re-configuring). When it runs out
41 of commits, it gets another job from the builder and starts again with that
44 Clearly the builder threads could work either way - they could check out a
45 commit and then built it for all boards. Using separate directories for each
46 commit/board pair they could leave their build product around afterwards
49 The intent behind building a single board for multiple commits, is to make
50 use of incremental builds. Since each commit is built incrementally from
51 the previous one, builds are faster. Reconfiguring for a different board
52 removes all intermediate object files.
54 Many threads can be working at once, but each has its own working directory.
55 When a thread finishes a build, it puts the output files into a result
58 The base directory used by buildman is normally '../<branch>', i.e.
59 a directory higher than the source repository and named after the branch
62 Within the base directory, we have one subdirectory for each commit. Within
63 that is one subdirectory for each board. Within that is the build output for
64 that commit/board combination.
66 Buildman also create working directories for each thread, in a .bm-work/
67 subdirectory in the base dir.
69 As an example, say we are building branch 'us-net' for boards 'sandbox' and
70 'seaboard', and say that us-net has two commits. We will have directories
73 us-net/ base directory
74 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
79 02_of_02_g4ed4ebc_net--Check-tftp-comp/
85 00/ working directory for thread 0 (contains source checkout)
87 01/ working directory for thread 1
90 u-boot/ source directory
94 # Possible build outcomes
95 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
97 # Translate a commit subject into a valid filename (and handle unicode)
98 trans_valid_chars = str.maketrans('/: ', '---')
100 BASE_CONFIG_FILENAMES = [
101 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
104 EXTRA_CONFIG_FILENAMES = [
105 '.config', '.config-spl', '.config-tpl',
106 'autoconf.mk', 'autoconf-spl.mk', 'autoconf-tpl.mk',
107 'autoconf.h', 'autoconf-spl.h','autoconf-tpl.h',
111 """Holds information about configuration settings for a board."""
112 def __init__(self, config_filename, target):
115 for fname in config_filename:
116 self.config[fname] = {}
118 def Add(self, fname, key, value):
119 self.config[fname][key] = value
123 for fname in self.config:
124 for key, value in self.config[fname].items():
126 val = val ^ hash(key) & hash(value)
130 """Holds information about environment variables for a board."""
131 def __init__(self, target):
133 self.environment = {}
135 def Add(self, key, value):
136 self.environment[key] = value
139 """Class for building U-Boot for a particular commit.
141 Public members: (many should ->private)
142 already_done: Number of builds already completed
143 base_dir: Base directory to use for builder
144 checkout: True to check out source, False to skip that step.
145 This is used for testing.
146 col: terminal.Color() object
147 count: Number of commits to build
148 do_make: Method to call to invoke Make
149 fail: Number of builds that failed due to error
150 force_build: Force building even if a build already exists
151 force_config_on_failure: If a commit fails for a board, disable
152 incremental building for the next commit we build for that
153 board, so that we will see all warnings/errors again.
154 force_build_failures: If a previously-built build (i.e. built on
155 a previous run of buildman) is marked as failed, rebuild it.
156 git_dir: Git directory containing source repository
157 last_line_len: Length of the last line we printed (used for erasing
158 it with new progress information)
159 num_jobs: Number of jobs to run at once (passed to make as -j)
160 num_threads: Number of builder threads to run
161 out_queue: Queue of results to process
162 re_make_err: Compiled regular expression for ignore_lines
163 queue: Queue of jobs to run
164 threads: List of active threads
165 toolchains: Toolchains object to use for building
166 upto: Current commit number we are building (0.count-1)
167 warned: Number of builds that produced at least one warning
168 force_reconfig: Reconfigure U-Boot on each comiit. This disables
169 incremental building, where buildman reconfigures on the first
170 commit for a baord, and then just does an incremental build for
171 the following commits. In fact buildman will reconfigure and
172 retry for any failing commits, so generally the only effect of
173 this option is to slow things down.
174 in_tree: Build U-Boot in-tree instead of specifying an output
175 directory separate from the source code. This option is really
176 only useful for testing in-tree builds.
177 work_in_output: Use the output directory as the work directory and
178 don't write to a separate output directory.
181 _base_board_dict: Last-summarised Dict of boards
182 _base_err_lines: Last-summarised list of errors
183 _base_warn_lines: Last-summarised list of warnings
184 _build_period_us: Time taken for a single build (float object).
185 _complete_delay: Expected delay until completion (timedelta)
186 _next_delay_update: Next time we plan to display a progress update
188 _show_unknown: Show unknown boards (those not built) in summary
189 _timestamps: List of timestamps for the completion of the last
190 last _timestamp_count builds. Each is a datetime object.
191 _timestamp_count: Number of timestamps to keep in our list.
192 _working_dir: Base working directory containing all threads
195 """Records a build outcome for a single make invocation
198 rc: Outcome value (OUTCOME_...)
199 err_lines: List of error lines or [] if none
200 sizes: Dictionary of image size information, keyed by filename
201 - Each value is itself a dictionary containing
202 values for 'text', 'data' and 'bss', being the integer
203 size in bytes of each section.
204 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
205 value is itself a dictionary:
207 value: Size of function in bytes
208 config: Dictionary keyed by filename - e.g. '.config'. Each
209 value is itself a dictionary:
212 environment: Dictionary keyed by environment variable, Each
213 value is the value of environment variable.
215 def __init__(self, rc, err_lines, sizes, func_sizes, config,
218 self.err_lines = err_lines
220 self.func_sizes = func_sizes
222 self.environment = environment
224 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
225 gnu_make='make', checkout=True, show_unknown=True, step=1,
226 no_subdirs=False, full_path=False, verbose_build=False,
227 incremental=False, per_board_out_dir=False,
228 config_only=False, squash_config_y=False,
229 warnings_as_errors=False, work_in_output=False):
230 """Create a new Builder object
233 toolchains: Toolchains object to use for building
234 base_dir: Base directory to use for builder
235 git_dir: Git directory containing source repository
236 num_threads: Number of builder threads to run
237 num_jobs: Number of jobs to run at once (passed to make as -j)
238 gnu_make: the command name of GNU Make.
239 checkout: True to check out source, False to skip that step.
240 This is used for testing.
241 show_unknown: Show unknown boards (those not built) in summary
242 step: 1 to process every commit, n to process every nth commit
243 no_subdirs: Don't create subdirectories when building current
244 source for a single board
245 full_path: Return the full path in CROSS_COMPILE and don't set
247 verbose_build: Run build with V=1 and don't use 'make -s'
248 incremental: Always perform incremental builds; don't run make
249 mrproper when configuring
250 per_board_out_dir: Build in a separate persistent directory per
251 board rather than a thread-specific directory
252 config_only: Only configure each build, don't build it
253 squash_config_y: Convert CONFIG options with the value 'y' to '1'
254 warnings_as_errors: Treat all compiler warnings as errors
255 work_in_output: Use the output directory as the work directory and
256 don't write to a separate output directory.
258 self.toolchains = toolchains
259 self.base_dir = base_dir
261 self._working_dir = base_dir
263 self._working_dir = os.path.join(base_dir, '.bm-work')
265 self.do_make = self.Make
266 self.gnu_make = gnu_make
267 self.checkout = checkout
268 self.num_threads = num_threads
269 self.num_jobs = num_jobs
270 self.already_done = 0
271 self.force_build = False
272 self.git_dir = git_dir
273 self._show_unknown = show_unknown
274 self._timestamp_count = 10
275 self._build_period_us = None
276 self._complete_delay = None
277 self._next_delay_update = datetime.now()
278 self.force_config_on_failure = True
279 self.force_build_failures = False
280 self.force_reconfig = False
283 self._error_lines = 0
284 self.no_subdirs = no_subdirs
285 self.full_path = full_path
286 self.verbose_build = verbose_build
287 self.config_only = config_only
288 self.squash_config_y = squash_config_y
289 self.config_filenames = BASE_CONFIG_FILENAMES
290 self.work_in_output = work_in_output
291 if not self.squash_config_y:
292 self.config_filenames += EXTRA_CONFIG_FILENAMES
294 self.warnings_as_errors = warnings_as_errors
295 self.col = terminal.Color()
297 self._re_function = re.compile('(.*): In function.*')
298 self._re_files = re.compile('In file included from.*')
299 self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
300 self._re_dtb_warning = re.compile('(.*): Warning .*')
301 self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
303 self.queue = queue.Queue()
304 self.out_queue = queue.Queue()
305 for i in range(self.num_threads):
306 t = builderthread.BuilderThread(self, i, incremental,
310 self.threads.append(t)
312 self.last_line_len = 0
313 t = builderthread.ResultThread(self)
316 self.threads.append(t)
318 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
319 self.re_make_err = re.compile('|'.join(ignore_lines))
321 # Handle existing graceful with SIGINT / Ctrl-C
322 signal.signal(signal.SIGINT, self.signal_handler)
325 """Get rid of all threads created by the builder"""
326 for t in self.threads:
329 def signal_handler(self, signal, frame):
332 def SetDisplayOptions(self, show_errors=False, show_sizes=False,
333 show_detail=False, show_bloat=False,
334 list_error_boards=False, show_config=False,
335 show_environment=False):
336 """Setup display options for the builder.
338 show_errors: True to show summarised error/warning info
339 show_sizes: Show size deltas
340 show_detail: Show size delta detail for each board if show_sizes
341 show_bloat: Show detail for each function
342 list_error_boards: Show the boards which caused each error/warning
343 show_config: Show config deltas
344 show_environment: Show environment deltas
346 self._show_errors = show_errors
347 self._show_sizes = show_sizes
348 self._show_detail = show_detail
349 self._show_bloat = show_bloat
350 self._list_error_boards = list_error_boards
351 self._show_config = show_config
352 self._show_environment = show_environment
354 def _AddTimestamp(self):
355 """Add a new timestamp to the list and record the build period.
357 The build period is the length of time taken to perform a single
358 build (one board, one commit).
361 self._timestamps.append(now)
362 count = len(self._timestamps)
363 delta = self._timestamps[-1] - self._timestamps[0]
364 seconds = delta.total_seconds()
366 # If we have enough data, estimate build period (time taken for a
367 # single build) and therefore completion time.
368 if count > 1 and self._next_delay_update < now:
369 self._next_delay_update = now + timedelta(seconds=2)
371 self._build_period = float(seconds) / count
372 todo = self.count - self.upto
373 self._complete_delay = timedelta(microseconds=
374 self._build_period * todo * 1000000)
376 self._complete_delay -= timedelta(
377 microseconds=self._complete_delay.microseconds)
380 self._timestamps.popleft()
383 def ClearLine(self, length):
384 """Clear any characters on the current line
386 Make way for a new line of length 'length', by outputting enough
387 spaces to clear out the old line. Then remember the new length for
391 length: Length of new line, in characters
393 if length < self.last_line_len:
394 Print(' ' * (self.last_line_len - length), newline=False)
395 Print('\r', newline=False)
396 self.last_line_len = length
399 def SelectCommit(self, commit, checkout=True):
400 """Checkout the selected commit for this build
403 if checkout and self.checkout:
404 gitutil.Checkout(commit.hash)
406 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
410 commit: Commit object that is being built
411 brd: Board object that is being built
412 stage: Stage that we are at (mrproper, config, build)
413 cwd: Directory where make should be run
414 args: Arguments to pass to make
415 kwargs: Arguments to pass to command.RunPipe()
417 cmd = [self.gnu_make] + list(args)
418 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
419 cwd=cwd, raise_on_error=False, infile='/dev/null', **kwargs)
420 if self.verbose_build:
421 result.stdout = '%s\n' % (' '.join(cmd)) + result.stdout
422 result.combined = '%s\n' % (' '.join(cmd)) + result.combined
425 def ProcessResult(self, result):
426 """Process the result of a build, showing progress information
429 result: A CommandResult object, which indicates the result for
432 col = terminal.Color()
434 target = result.brd.target
437 if result.return_code != 0:
441 if result.already_done:
442 self.already_done += 1
444 Print('\r', newline=False)
446 boards_selected = {target : result.brd}
447 self.ResetResultSummary(boards_selected)
448 self.ProduceResultSummary(result.commit_upto, self.commits,
451 target = '(starting)'
453 # Display separate counts for ok, warned and fail
454 ok = self.upto - self.warned - self.fail
455 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
456 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
457 line += self.col.Color(self.col.RED, '%5d' % self.fail)
459 name = ' /%-5d ' % self.count
461 # Add our current completion time estimate
463 if self._complete_delay:
464 name += '%s : ' % self._complete_delay
465 # When building all boards for a commit, we can print a commit
467 if result and result.commit_upto is None:
468 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
472 Print(line + name, newline=False)
473 length = 16 + len(name)
474 self.ClearLine(length)
476 def _GetOutputDir(self, commit_upto):
477 """Get the name of the output directory for a commit number
479 The output directory is typically .../<branch>/<commit>.
482 commit_upto: Commit number to use (0..self.count-1)
486 commit = self.commits[commit_upto]
487 subject = commit.subject.translate(trans_valid_chars)
488 # See _GetOutputSpaceRemovals() which parses this name
489 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
490 self.commit_count, commit.hash, subject[:20]))
491 elif not self.no_subdirs:
492 commit_dir = 'current'
495 return os.path.join(self.base_dir, commit_dir)
497 def GetBuildDir(self, commit_upto, target):
498 """Get the name of the build directory for a commit number
500 The build directory is typically .../<branch>/<commit>/<target>.
503 commit_upto: Commit number to use (0..self.count-1)
506 output_dir = self._GetOutputDir(commit_upto)
507 return os.path.join(output_dir, target)
509 def GetDoneFile(self, commit_upto, target):
510 """Get the name of the done file for a commit number
513 commit_upto: Commit number to use (0..self.count-1)
516 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
518 def GetSizesFile(self, commit_upto, target):
519 """Get the name of the sizes file for a commit number
522 commit_upto: Commit number to use (0..self.count-1)
525 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
527 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
528 """Get the name of the funcsizes file for a commit number and ELF file
531 commit_upto: Commit number to use (0..self.count-1)
533 elf_fname: Filename of elf image
535 return os.path.join(self.GetBuildDir(commit_upto, target),
536 '%s.sizes' % elf_fname.replace('/', '-'))
538 def GetObjdumpFile(self, commit_upto, target, elf_fname):
539 """Get the name of the objdump file for a commit number and ELF file
542 commit_upto: Commit number to use (0..self.count-1)
544 elf_fname: Filename of elf image
546 return os.path.join(self.GetBuildDir(commit_upto, target),
547 '%s.objdump' % elf_fname.replace('/', '-'))
549 def GetErrFile(self, commit_upto, target):
550 """Get the name of the err file for a commit number
553 commit_upto: Commit number to use (0..self.count-1)
556 output_dir = self.GetBuildDir(commit_upto, target)
557 return os.path.join(output_dir, 'err')
559 def FilterErrors(self, lines):
560 """Filter out errors in which we have no interest
562 We should probably use map().
565 lines: List of error lines, each a string
567 New list with only interesting lines included
571 if not self.re_make_err.search(line):
572 out_lines.append(line)
575 def ReadFuncSizes(self, fname, fd):
576 """Read function sizes from the output of 'nm'
579 fd: File containing data to read
580 fname: Filename we are reading from (just for errors)
583 Dictionary containing size of each function in bytes, indexed by
587 for line in fd.readlines():
590 size, type, name = line[:-1].split()
592 Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
595 # function names begin with '.' on 64-bit powerpc
597 name = 'static.' + name.split('.')[0]
598 sym[name] = sym.get(name, 0) + int(size, 16)
601 def _ProcessConfig(self, fname):
602 """Read in a .config, autoconf.mk or autoconf.h file
604 This function handles all config file types. It ignores comments and
605 any #defines which don't start with CONFIG_.
608 fname: Filename to read
612 key: Config name (e.g. CONFIG_DM)
613 value: Config value (e.g. 1)
616 if os.path.exists(fname):
617 with open(fname) as fd:
620 if line.startswith('#define'):
621 values = line[8:].split(' ', 1)
626 value = '1' if self.squash_config_y else ''
627 if not key.startswith('CONFIG_'):
629 elif not line or line[0] in ['#', '*', '/']:
632 key, value = line.split('=', 1)
633 if self.squash_config_y and value == 'y':
638 def _ProcessEnvironment(self, fname):
639 """Read in a uboot.env file
641 This function reads in environment variables from a file.
644 fname: Filename to read
648 key: environment variable (e.g. bootlimit)
649 value: value of environment variable (e.g. 1)
652 if os.path.exists(fname):
653 with open(fname) as fd:
654 for line in fd.read().split('\0'):
656 key, value = line.split('=', 1)
657 environment[key] = value
659 # ignore lines we can't parse
663 def GetBuildOutcome(self, commit_upto, target, read_func_sizes,
664 read_config, read_environment):
665 """Work out the outcome of a build.
668 commit_upto: Commit number to check (0..n-1)
669 target: Target board to check
670 read_func_sizes: True to read function size information
671 read_config: True to read .config and autoconf.h files
672 read_environment: True to read uboot.env files
677 done_file = self.GetDoneFile(commit_upto, target)
678 sizes_file = self.GetSizesFile(commit_upto, target)
683 if os.path.exists(done_file):
684 with open(done_file, 'r') as fd:
686 return_code = int(fd.readline())
688 # The file may be empty due to running out of disk space.
692 err_file = self.GetErrFile(commit_upto, target)
693 if os.path.exists(err_file):
694 with open(err_file, 'r') as fd:
695 err_lines = self.FilterErrors(fd.readlines())
697 # Decide whether the build was ok, failed or created warnings
705 # Convert size information to our simple format
706 if os.path.exists(sizes_file):
707 with open(sizes_file, 'r') as fd:
708 for line in fd.readlines():
709 values = line.split()
712 rodata = int(values[6], 16)
714 'all' : int(values[0]) + int(values[1]) +
716 'text' : int(values[0]) - rodata,
717 'data' : int(values[1]),
718 'bss' : int(values[2]),
721 sizes[values[5]] = size_dict
724 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
725 for fname in glob.glob(pattern):
726 with open(fname, 'r') as fd:
727 dict_name = os.path.basename(fname).replace('.sizes',
729 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
732 output_dir = self.GetBuildDir(commit_upto, target)
733 for name in self.config_filenames:
734 fname = os.path.join(output_dir, name)
735 config[name] = self._ProcessConfig(fname)
738 output_dir = self.GetBuildDir(commit_upto, target)
739 fname = os.path.join(output_dir, 'uboot.env')
740 environment = self._ProcessEnvironment(fname)
742 return Builder.Outcome(rc, err_lines, sizes, func_sizes, config,
745 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {}, {}, {})
747 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes,
748 read_config, read_environment):
749 """Calculate a summary of the results of building a commit.
752 board_selected: Dict containing boards to summarise
753 commit_upto: Commit number to summarize (0..self.count-1)
754 read_func_sizes: True to read function size information
755 read_config: True to read .config and autoconf.h files
756 read_environment: True to read uboot.env files
760 Dict containing boards which passed building this commit.
761 keyed by board.target
762 List containing a summary of error lines
763 Dict keyed by error line, containing a list of the Board
764 objects with that error
765 List containing a summary of warning lines
766 Dict keyed by error line, containing a list of the Board
767 objects with that warning
768 Dictionary keyed by board.target. Each value is a dictionary:
769 key: filename - e.g. '.config'
770 value is itself a dictionary:
773 Dictionary keyed by board.target. Each value is a dictionary:
774 key: environment variable
775 value: value of environment variable
777 def AddLine(lines_summary, lines_boards, line, board):
779 if line in lines_boards:
780 lines_boards[line].append(board)
782 lines_boards[line] = [board]
783 lines_summary.append(line)
786 err_lines_summary = []
787 err_lines_boards = {}
788 warn_lines_summary = []
789 warn_lines_boards = {}
793 for board in boards_selected.values():
794 outcome = self.GetBuildOutcome(commit_upto, board.target,
795 read_func_sizes, read_config,
797 board_dict[board.target] = outcome
799 last_was_warning = False
800 for line in outcome.err_lines:
802 if (self._re_function.match(line) or
803 self._re_files.match(line)):
806 is_warning = (self._re_warning.match(line) or
807 self._re_dtb_warning.match(line))
808 is_note = self._re_note.match(line)
809 if is_warning or (last_was_warning and is_note):
811 AddLine(warn_lines_summary, warn_lines_boards,
813 AddLine(warn_lines_summary, warn_lines_boards,
817 AddLine(err_lines_summary, err_lines_boards,
819 AddLine(err_lines_summary, err_lines_boards,
821 last_was_warning = is_warning
823 tconfig = Config(self.config_filenames, board.target)
824 for fname in self.config_filenames:
826 for key, value in outcome.config[fname].items():
827 tconfig.Add(fname, key, value)
828 config[board.target] = tconfig
830 tenvironment = Environment(board.target)
831 if outcome.environment:
832 for key, value in outcome.environment.items():
833 tenvironment.Add(key, value)
834 environment[board.target] = tenvironment
836 return (board_dict, err_lines_summary, err_lines_boards,
837 warn_lines_summary, warn_lines_boards, config, environment)
839 def AddOutcome(self, board_dict, arch_list, changes, char, color):
840 """Add an output to our list of outcomes for each architecture
842 This simple function adds failing boards (changes) to the
843 relevant architecture string, so we can print the results out
844 sorted by architecture.
847 board_dict: Dict containing all boards
848 arch_list: Dict keyed by arch name. Value is a string containing
849 a list of board names which failed for that arch.
850 changes: List of boards to add to arch_list
851 color: terminal.Colour object
854 for target in changes:
855 if target in board_dict:
856 arch = board_dict[target].arch
859 str = self.col.Color(color, ' ' + target)
860 if not arch in done_arch:
861 str = ' %s %s' % (self.col.Color(color, char), str)
862 done_arch[arch] = True
863 if not arch in arch_list:
864 arch_list[arch] = str
866 arch_list[arch] += str
869 def ColourNum(self, num):
870 color = self.col.RED if num > 0 else self.col.GREEN
873 return self.col.Color(color, str(num))
875 def ResetResultSummary(self, board_selected):
876 """Reset the results summary ready for use.
878 Set up the base board list to be all those selected, and set the
879 error lines to empty.
881 Following this, calls to PrintResultSummary() will use this
882 information to work out what has changed.
885 board_selected: Dict containing boards to summarise, keyed by
888 self._base_board_dict = {}
889 for board in board_selected:
890 self._base_board_dict[board] = Builder.Outcome(0, [], [], {}, {},
892 self._base_err_lines = []
893 self._base_warn_lines = []
894 self._base_err_line_boards = {}
895 self._base_warn_line_boards = {}
896 self._base_config = None
897 self._base_environment = None
899 def PrintFuncSizeDetail(self, fname, old, new):
900 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
901 delta, common = [], {}
908 if name not in common:
911 delta.append([-old[name], name])
914 if name not in common:
917 delta.append([new[name], name])
920 diff = new.get(name, 0) - old.get(name, 0)
922 grow, up = grow + 1, up + diff
924 shrink, down = shrink + 1, down - diff
925 delta.append([diff, name])
930 args = [add, -remove, grow, -shrink, up, -down, up - down]
931 if max(args) == 0 and min(args) == 0:
933 args = [self.ColourNum(x) for x in args]
935 Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
936 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
937 Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
939 for diff, name in delta:
941 color = self.col.RED if diff > 0 else self.col.GREEN
942 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
943 old.get(name, '-'), new.get(name,'-'), diff)
944 Print(msg, colour=color)
947 def PrintSizeDetail(self, target_list, show_bloat):
948 """Show details size information for each board
951 target_list: List of targets, each a dict containing:
952 'target': Target name
953 'total_diff': Total difference in bytes across all areas
954 <part_name>: Difference for that part
955 show_bloat: Show detail for each function
957 targets_by_diff = sorted(target_list, reverse=True,
958 key=lambda x: x['_total_diff'])
959 for result in targets_by_diff:
960 printed_target = False
961 for name in sorted(result):
963 if name.startswith('_'):
966 color = self.col.RED if diff > 0 else self.col.GREEN
967 msg = ' %s %+d' % (name, diff)
968 if not printed_target:
969 Print('%10s %-15s:' % ('', result['_target']),
971 printed_target = True
972 Print(msg, colour=color, newline=False)
976 target = result['_target']
977 outcome = result['_outcome']
978 base_outcome = self._base_board_dict[target]
979 for fname in outcome.func_sizes:
980 self.PrintFuncSizeDetail(fname,
981 base_outcome.func_sizes[fname],
982 outcome.func_sizes[fname])
985 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
987 """Print a summary of image sizes broken down by section.
989 The summary takes the form of one line per architecture. The
990 line contains deltas for each of the sections (+ means the section
991 got bigger, - means smaller). The numbers are the average number
992 of bytes that a board in this section increased by.
995 powerpc: (622 boards) text -0.0
996 arm: (285 boards) text -0.0
997 nds32: (3 boards) text -8.0
1000 board_selected: Dict containing boards to summarise, keyed by
1002 board_dict: Dict containing boards for which we built this
1003 commit, keyed by board.target. The value is an Outcome object.
1004 show_detail: Show size delta detail for each board
1005 show_bloat: Show detail for each function
1010 # Calculate changes in size for different image parts
1011 # The previous sizes are in Board.sizes, for each board
1012 for target in board_dict:
1013 if target not in board_selected:
1015 base_sizes = self._base_board_dict[target].sizes
1016 outcome = board_dict[target]
1017 sizes = outcome.sizes
1019 # Loop through the list of images, creating a dict of size
1020 # changes for each image/part. We end up with something like
1021 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1022 # which means that U-Boot data increased by 5 bytes and SPL
1023 # text decreased by 4.
1024 err = {'_target' : target}
1026 if image in base_sizes:
1027 base_image = base_sizes[image]
1028 # Loop through the text, data, bss parts
1029 for part in sorted(sizes[image]):
1030 diff = sizes[image][part] - base_image[part]
1033 if image == 'u-boot':
1036 name = image + ':' + part
1038 arch = board_selected[target].arch
1039 if not arch in arch_count:
1040 arch_count[arch] = 1
1042 arch_count[arch] += 1
1044 pass # Only add to our list when we have some stats
1045 elif not arch in arch_list:
1046 arch_list[arch] = [err]
1048 arch_list[arch].append(err)
1050 # We now have a list of image size changes sorted by arch
1051 # Print out a summary of these
1052 for arch, target_list in arch_list.items():
1053 # Get total difference for each type
1055 for result in target_list:
1057 for name, diff in result.items():
1058 if name.startswith('_'):
1062 totals[name] += diff
1065 result['_total_diff'] = total
1066 result['_outcome'] = board_dict[result['_target']]
1068 count = len(target_list)
1069 printed_arch = False
1070 for name in sorted(totals):
1073 # Display the average difference in this name for this
1075 avg_diff = float(diff) / count
1076 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1077 msg = ' %s %+1.1f' % (name, avg_diff)
1078 if not printed_arch:
1079 Print('%10s: (for %d/%d boards)' % (arch, count,
1080 arch_count[arch]), newline=False)
1082 Print(msg, colour=color, newline=False)
1087 self.PrintSizeDetail(target_list, show_bloat)
1090 def PrintResultSummary(self, board_selected, board_dict, err_lines,
1091 err_line_boards, warn_lines, warn_line_boards,
1092 config, environment, show_sizes, show_detail,
1093 show_bloat, show_config, show_environment):
1094 """Compare results with the base results and display delta.
1096 Only boards mentioned in board_selected will be considered. This
1097 function is intended to be called repeatedly with the results of
1098 each commit. It therefore shows a 'diff' between what it saw in
1099 the last call and what it sees now.
1102 board_selected: Dict containing boards to summarise, keyed by
1104 board_dict: Dict containing boards for which we built this
1105 commit, keyed by board.target. The value is an Outcome object.
1106 err_lines: A list of errors for this commit, or [] if there is
1107 none, or we don't want to print errors
1108 err_line_boards: Dict keyed by error line, containing a list of
1109 the Board objects with that error
1110 warn_lines: A list of warnings for this commit, or [] if there is
1111 none, or we don't want to print errors
1112 warn_line_boards: Dict keyed by warning line, containing a list of
1113 the Board objects with that warning
1114 config: Dictionary keyed by filename - e.g. '.config'. Each
1115 value is itself a dictionary:
1118 environment: Dictionary keyed by environment variable, Each
1119 value is the value of environment variable.
1120 show_sizes: Show image size deltas
1121 show_detail: Show size delta detail for each board if show_sizes
1122 show_bloat: Show detail for each function
1123 show_config: Show config changes
1124 show_environment: Show environment changes
1126 def _BoardList(line, line_boards):
1127 """Helper function to get a line of boards containing a line
1130 line: Error line to search for
1132 String containing a list of boards with that error line, or
1133 '' if the user has not requested such a list
1135 if self._list_error_boards:
1137 for board in line_boards[line]:
1138 if not board.target in names:
1139 names.append(board.target)
1140 names_str = '(%s) ' % ','.join(names)
1145 def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
1150 if line not in base_lines:
1151 worse_lines.append(char + '+' +
1152 _BoardList(line, line_boards) + line)
1153 for line in base_lines:
1154 if line not in lines:
1155 better_lines.append(char + '-' +
1156 _BoardList(line, base_line_boards) + line)
1157 return better_lines, worse_lines
1159 def _CalcConfig(delta, name, config):
1160 """Calculate configuration changes
1163 delta: Type of the delta, e.g. '+'
1164 name: name of the file which changed (e.g. .config)
1165 config: configuration change dictionary
1169 String containing the configuration changes which can be
1173 for key in sorted(config.keys()):
1174 out += '%s=%s ' % (key, config[key])
1175 return '%s %s: %s' % (delta, name, out)
1177 def _AddConfig(lines, name, config_plus, config_minus, config_change):
1178 """Add changes in configuration to a list
1181 lines: list to add to
1182 name: config file name
1183 config_plus: configurations added, dictionary
1186 config_minus: configurations removed, dictionary
1189 config_change: configurations changed, dictionary
1194 lines.append(_CalcConfig('+', name, config_plus))
1196 lines.append(_CalcConfig('-', name, config_minus))
1198 lines.append(_CalcConfig('c', name, config_change))
1200 def _OutputConfigInfo(lines):
1205 col = self.col.GREEN
1206 elif line[0] == '-':
1208 elif line[0] == 'c':
1209 col = self.col.YELLOW
1210 Print(' ' + line, newline=True, colour=col)
1213 ok_boards = [] # List of boards fixed since last commit
1214 warn_boards = [] # List of boards with warnings since last commit
1215 err_boards = [] # List of new broken boards since last commit
1216 new_boards = [] # List of boards that didn't exist last time
1217 unknown_boards = [] # List of boards that were not built
1219 for target in board_dict:
1220 if target not in board_selected:
1223 # If the board was built last time, add its outcome to a list
1224 if target in self._base_board_dict:
1225 base_outcome = self._base_board_dict[target].rc
1226 outcome = board_dict[target]
1227 if outcome.rc == OUTCOME_UNKNOWN:
1228 unknown_boards.append(target)
1229 elif outcome.rc < base_outcome:
1230 if outcome.rc == OUTCOME_WARNING:
1231 warn_boards.append(target)
1233 ok_boards.append(target)
1234 elif outcome.rc > base_outcome:
1235 if outcome.rc == OUTCOME_WARNING:
1236 warn_boards.append(target)
1238 err_boards.append(target)
1240 new_boards.append(target)
1242 # Get a list of errors that have appeared, and disappeared
1243 better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
1244 self._base_err_line_boards, err_lines, err_line_boards, '')
1245 better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
1246 self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
1248 # Display results by arch
1249 if any((ok_boards, warn_boards, err_boards, unknown_boards, new_boards,
1250 worse_err, better_err, worse_warn, better_warn)):
1252 self.AddOutcome(board_selected, arch_list, ok_boards, '',
1254 self.AddOutcome(board_selected, arch_list, warn_boards, 'w+',
1256 self.AddOutcome(board_selected, arch_list, err_boards, '+',
1258 self.AddOutcome(board_selected, arch_list, new_boards, '*', self.col.BLUE)
1259 if self._show_unknown:
1260 self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
1262 for arch, target_list in arch_list.items():
1263 Print('%10s: %s' % (arch, target_list))
1264 self._error_lines += 1
1266 Print('\n'.join(better_err), colour=self.col.GREEN)
1267 self._error_lines += 1
1269 Print('\n'.join(worse_err), colour=self.col.RED)
1270 self._error_lines += 1
1272 Print('\n'.join(better_warn), colour=self.col.CYAN)
1273 self._error_lines += 1
1275 Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
1276 self._error_lines += 1
1279 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1282 if show_environment and self._base_environment:
1285 for target in board_dict:
1286 if target not in board_selected:
1289 tbase = self._base_environment[target]
1290 tenvironment = environment[target]
1291 environment_plus = {}
1292 environment_minus = {}
1293 environment_change = {}
1294 base = tbase.environment
1295 for key, value in tenvironment.environment.items():
1297 environment_plus[key] = value
1298 for key, value in base.items():
1299 if key not in tenvironment.environment:
1300 environment_minus[key] = value
1301 for key, value in base.items():
1302 new_value = tenvironment.environment.get(key)
1303 if new_value and value != new_value:
1304 desc = '%s -> %s' % (value, new_value)
1305 environment_change[key] = desc
1307 _AddConfig(lines, target, environment_plus, environment_minus,
1310 _OutputConfigInfo(lines)
1312 if show_config and self._base_config:
1314 arch_config_plus = {}
1315 arch_config_minus = {}
1316 arch_config_change = {}
1319 for target in board_dict:
1320 if target not in board_selected:
1322 arch = board_selected[target].arch
1323 if arch not in arch_list:
1324 arch_list.append(arch)
1326 for arch in arch_list:
1327 arch_config_plus[arch] = {}
1328 arch_config_minus[arch] = {}
1329 arch_config_change[arch] = {}
1330 for name in self.config_filenames:
1331 arch_config_plus[arch][name] = {}
1332 arch_config_minus[arch][name] = {}
1333 arch_config_change[arch][name] = {}
1335 for target in board_dict:
1336 if target not in board_selected:
1339 arch = board_selected[target].arch
1341 all_config_plus = {}
1342 all_config_minus = {}
1343 all_config_change = {}
1344 tbase = self._base_config[target]
1345 tconfig = config[target]
1347 for name in self.config_filenames:
1348 if not tconfig.config[name]:
1353 base = tbase.config[name]
1354 for key, value in tconfig.config[name].items():
1356 config_plus[key] = value
1357 all_config_plus[key] = value
1358 for key, value in base.items():
1359 if key not in tconfig.config[name]:
1360 config_minus[key] = value
1361 all_config_minus[key] = value
1362 for key, value in base.items():
1363 new_value = tconfig.config.get(key)
1364 if new_value and value != new_value:
1365 desc = '%s -> %s' % (value, new_value)
1366 config_change[key] = desc
1367 all_config_change[key] = desc
1369 arch_config_plus[arch][name].update(config_plus)
1370 arch_config_minus[arch][name].update(config_minus)
1371 arch_config_change[arch][name].update(config_change)
1373 _AddConfig(lines, name, config_plus, config_minus,
1375 _AddConfig(lines, 'all', all_config_plus, all_config_minus,
1377 summary[target] = '\n'.join(lines)
1379 lines_by_target = {}
1380 for target, lines in summary.items():
1381 if lines in lines_by_target:
1382 lines_by_target[lines].append(target)
1384 lines_by_target[lines] = [target]
1386 for arch in arch_list:
1391 for name in self.config_filenames:
1392 all_plus.update(arch_config_plus[arch][name])
1393 all_minus.update(arch_config_minus[arch][name])
1394 all_change.update(arch_config_change[arch][name])
1395 _AddConfig(lines, name, arch_config_plus[arch][name],
1396 arch_config_minus[arch][name],
1397 arch_config_change[arch][name])
1398 _AddConfig(lines, 'all', all_plus, all_minus, all_change)
1399 #arch_summary[target] = '\n'.join(lines)
1402 _OutputConfigInfo(lines)
1404 for lines, targets in lines_by_target.items():
1407 Print('%s :' % ' '.join(sorted(targets)))
1408 _OutputConfigInfo(lines.split('\n'))
1411 # Save our updated information for the next call to this function
1412 self._base_board_dict = board_dict
1413 self._base_err_lines = err_lines
1414 self._base_warn_lines = warn_lines
1415 self._base_err_line_boards = err_line_boards
1416 self._base_warn_line_boards = warn_line_boards
1417 self._base_config = config
1418 self._base_environment = environment
1420 # Get a list of boards that did not get built, if needed
1422 for board in board_selected:
1423 if not board in board_dict:
1424 not_built.append(board)
1426 Print("Boards not built (%d): %s" % (len(not_built),
1427 ', '.join(not_built)))
1429 def ProduceResultSummary(self, commit_upto, commits, board_selected):
1430 (board_dict, err_lines, err_line_boards, warn_lines,
1431 warn_line_boards, config, environment) = self.GetResultSummary(
1432 board_selected, commit_upto,
1433 read_func_sizes=self._show_bloat,
1434 read_config=self._show_config,
1435 read_environment=self._show_environment)
1437 msg = '%02d: %s' % (commit_upto + 1,
1438 commits[commit_upto].subject)
1439 Print(msg, colour=self.col.BLUE)
1440 self.PrintResultSummary(board_selected, board_dict,
1441 err_lines if self._show_errors else [], err_line_boards,
1442 warn_lines if self._show_errors else [], warn_line_boards,
1443 config, environment, self._show_sizes, self._show_detail,
1444 self._show_bloat, self._show_config, self._show_environment)
1446 def ShowSummary(self, commits, board_selected):
1447 """Show a build summary for U-Boot for a given board list.
1449 Reset the result summary, then repeatedly call GetResultSummary on
1450 each commit's results, then display the differences we see.
1453 commit: Commit objects to summarise
1454 board_selected: Dict containing boards to summarise
1456 self.commit_count = len(commits) if commits else 1
1457 self.commits = commits
1458 self.ResetResultSummary(board_selected)
1459 self._error_lines = 0
1461 for commit_upto in range(0, self.commit_count, self._step):
1462 self.ProduceResultSummary(commit_upto, commits, board_selected)
1463 if not self._error_lines:
1464 Print('(no errors to report)', colour=self.col.GREEN)
1467 def SetupBuild(self, board_selected, commits):
1468 """Set up ready to start a build.
1471 board_selected: Selected boards to build
1472 commits: Selected commits to build
1474 # First work out how many commits we will build
1475 count = (self.commit_count + self._step - 1) // self._step
1476 self.count = len(board_selected) * count
1477 self.upto = self.warned = self.fail = 0
1478 self._timestamps = collections.deque()
1480 def GetThreadDir(self, thread_num):
1481 """Get the directory path to the working dir for a thread.
1484 thread_num: Number of thread to check.
1486 if self.work_in_output:
1487 return self._working_dir
1488 return os.path.join(self._working_dir, '%02d' % thread_num)
1490 def _PrepareThread(self, thread_num, setup_git):
1491 """Prepare the working directory for a thread.
1493 This clones or fetches the repo into the thread's work directory.
1496 thread_num: Thread number (0, 1, ...)
1497 setup_git: True to set up a git repo clone
1499 thread_dir = self.GetThreadDir(thread_num)
1500 builderthread.Mkdir(thread_dir)
1501 git_dir = os.path.join(thread_dir, '.git')
1503 # Clone the repo if it doesn't already exist
1504 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1505 # we have a private index but uses the origin repo's contents?
1506 if setup_git and self.git_dir:
1507 src_dir = os.path.abspath(self.git_dir)
1508 if os.path.exists(git_dir):
1509 gitutil.Fetch(git_dir, thread_dir)
1511 Print('\rCloning repo for thread %d' % thread_num,
1513 gitutil.Clone(src_dir, thread_dir)
1514 Print('\r%s\r' % (' ' * 30), newline=False)
1516 def _PrepareWorkingSpace(self, max_threads, setup_git):
1517 """Prepare the working directory for use.
1519 Set up the git repo for each thread.
1522 max_threads: Maximum number of threads we expect to need.
1523 setup_git: True to set up a git repo clone
1525 builderthread.Mkdir(self._working_dir)
1526 for thread in range(max_threads):
1527 self._PrepareThread(thread, setup_git)
1529 def _GetOutputSpaceRemovals(self):
1530 """Get the output directories ready to receive files.
1532 Figure out what needs to be deleted in the output directory before it
1533 can be used. We only delete old buildman directories which have the
1534 expected name pattern. See _GetOutputDir().
1537 List of full paths of directories to remove
1539 if not self.commits:
1542 for commit_upto in range(self.commit_count):
1543 dir_list.append(self._GetOutputDir(commit_upto))
1546 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1547 if dirname not in dir_list:
1548 leaf = dirname[len(self.base_dir) + 1:]
1549 m = re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
1551 to_remove.append(dirname)
1554 def _PrepareOutputSpace(self):
1555 """Get the output directories ready to receive files.
1557 We delete any output directories which look like ones we need to
1558 create. Having left over directories is confusing when the user wants
1559 to check the output manually.
1561 to_remove = self._GetOutputSpaceRemovals()
1563 Print('Removing %d old build directories...' % len(to_remove),
1565 for dirname in to_remove:
1566 shutil.rmtree(dirname)
1569 def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
1570 """Build all commits for a list of boards
1573 commits: List of commits to be build, each a Commit object
1574 boards_selected: Dict of selected boards, key is target name,
1575 value is Board object
1576 keep_outputs: True to save build output files
1577 verbose: Display build results as they are completed
1580 - number of boards that failed to build
1581 - number of boards that issued warnings
1583 self.commit_count = len(commits) if commits else 1
1584 self.commits = commits
1585 self._verbose = verbose
1587 self.ResetResultSummary(board_selected)
1588 builderthread.Mkdir(self.base_dir, parents = True)
1589 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
1590 commits is not None)
1591 self._PrepareOutputSpace()
1592 Print('\rStarting build...', newline=False)
1593 self.SetupBuild(board_selected, commits)
1594 self.ProcessResult(None)
1596 # Create jobs to build all commits for each board
1597 for brd in board_selected.values():
1598 job = builderthread.BuilderJob()
1600 job.commits = commits
1601 job.keep_outputs = keep_outputs
1602 job.work_in_output = self.work_in_output
1603 job.step = self._step
1606 term = threading.Thread(target=self.queue.join)
1607 term.setDaemon(True)
1609 while term.isAlive():
1612 # Wait until we have processed all output
1613 self.out_queue.join()
1616 return (self.fail, self.warned)