tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import board
  12 from buildman import bsettings
  13 from buildman import toolchain
  14 from buildman.builder import Builder
  15 from patman import command
  16 from patman import gitutil
  17 from patman import patchstream
  18 from patman import terminal
  19 from patman.terminal import Print
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def ShowToolchainPrefix(boards, toolchains):
  89     """Show information about a the tool chain used by one or more boards
  90
  91     The function checks that all boards use the same toolchain, then prints
  92     the correct value for CROSS_COMPILE.
  93
  94     Args:
  95         boards: Boards object containing selected boards
  96         toolchains: Toolchains object containing available toolchains
  97
  98     Return:
  99         None on success, string error message otherwise
 100     """
 101     boards = boards.GetSelectedDict()
 102     tc_set = set()
 103     for brd in boards.values():
 104         tc_set.add(toolchains.Select(brd.arch))
 105     if len(tc_set) != 1:
 106         return 'Supplied boards must share one toolchain'
 107         return False
 108     tc = tc_set.pop()
 109     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 110     return None
 111
 112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 113                clean_dir=False, test_thread_exceptions=False):
 114     """The main control code for buildman
 115
 116     Args:
 117         options: Command line options object
 118         args: Command line arguments (list of strings)
 119         toolchains: Toolchains to use - this should be a Toolchains()
 120                 object. If None, then it will be created and scanned
 121         make_func: Make function to use for the builder. This is called
 122                 to execute 'make'. If this is None, the normal function
 123                 will be used, which calls the 'make' tool with suitable
 124                 arguments. This setting is useful for tests.
 125         board: Boards() object to use, containing a list of available
 126                 boards. If this is None it will be created and scanned.
 127         clean_dir: Used for tests only, indicates that the existing output_dir
 128             should be removed before starting the build
 129         test_thread_exceptions: Uses for tests only, True to make the threads
 130             raise an exception instead of reporting their result. This simulates
 131             a failure in the code somewhere
 132     """
 133     global builder
 134
 135     if options.full_help:
 136         pager = os.getenv('PAGER')
 137         if not pager:
 138             pager = 'more'
 139         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 140                              'README')
 141         command.Run(pager, fname)
 142         return 0
 143
 144     gitutil.Setup()
 145     col = terminal.Color()
 146
 147     options.git_dir = os.path.join(options.git, '.git')
 148
 149     no_toolchains = toolchains is None
 150     if no_toolchains:
 151         toolchains = toolchain.Toolchains(options.override_toolchain)
 152
 153     if options.fetch_arch:
 154         if options.fetch_arch == 'list':
 155             sorted_list = toolchains.ListArchs()
 156             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 157                             ' '.join(sorted_list)))
 158             return 0
 159         else:
 160             fetch_arch = options.fetch_arch
 161             if fetch_arch == 'all':
 162                 fetch_arch = ','.join(toolchains.ListArchs())
 163                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 164                                 fetch_arch))
 165             for arch in fetch_arch.split(','):
 166                 print()
 167                 ret = toolchains.FetchAndInstall(arch)
 168                 if ret:
 169                     return ret
 170             return 0
 171
 172     if no_toolchains:
 173         toolchains.GetSettings()
 174         toolchains.Scan(options.list_tool_chains and options.verbose)
 175     if options.list_tool_chains:
 176         toolchains.List()
 177         print()
 178         return 0
 179
 180     if options.incremental:
 181         print(col.Color(col.RED,
 182                         'Warning: -I has been removed. See documentation'))
 183     if not options.output_dir:
 184         if options.work_in_output:
 185             sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
 186         options.output_dir = '..'
 187
 188     # Work out what subset of the boards we are building
 189     if not boards:
 190         if not os.path.exists(options.output_dir):
 191             os.makedirs(options.output_dir)
 192         board_file = os.path.join(options.output_dir, 'boards.cfg')
 193         our_path = os.path.dirname(os.path.realpath(__file__))
 194         genboardscfg = os.path.join(our_path, '../genboardscfg.py')
 195         if not os.path.exists(genboardscfg):
 196             genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 197         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 198         if status != 0:
 199             # Older versions don't support -q
 200             status = subprocess.call([genboardscfg, '-o', board_file])
 201             if status != 0:
 202                 sys.exit("Failed to generate boards.cfg")
 203
 204         boards = board.Boards()
 205         boards.ReadBoards(board_file)
 206
 207     exclude = []
 208     if options.exclude:
 209         for arg in options.exclude:
 210             exclude += arg.split(',')
 211
 212     if options.boards:
 213         requested_boards = []
 214         for b in options.boards:
 215             requested_boards += b.split(',')
 216     else:
 217         requested_boards = None
 218     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 219                                                        requested_boards)
 220     selected = boards.GetSelected()
 221     if not len(selected):
 222         sys.exit(col.Color(col.RED, 'No matching boards found'))
 223
 224     if options.print_prefix:
 225         err = ShowToolchainPrefix(boards, toolchains)
 226         if err:
 227             sys.exit(col.Color(col.RED, err))
 228         return 0
 229
 230     # Work out how many commits to build. We want to build everything on the
 231     # branch. We also build the upstream commit as a control so we can see
 232     # problems introduced by the first commit on the branch.
 233     count = options.count
 234     has_range = options.branch and '..' in options.branch
 235     if count == -1:
 236         if not options.branch:
 237             count = 1
 238         else:
 239             if has_range:
 240                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 241                                                          options.branch)
 242             else:
 243                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 244                                                           options.branch)
 245             if count is None:
 246                 sys.exit(col.Color(col.RED, msg))
 247             elif count == 0:
 248                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 249                                    options.branch))
 250             if msg:
 251                 print(col.Color(col.YELLOW, msg))
 252             count += 1   # Build upstream commit also
 253
 254     if not count:
 255         str = ("No commits found to process in branch '%s': "
 256                "set branch's upstream or use -c flag" % options.branch)
 257         sys.exit(col.Color(col.RED, str))
 258     if options.work_in_output:
 259         if len(selected) != 1:
 260             sys.exit(col.Color(col.RED,
 261                                '-w can only be used with a single board'))
 262         if count != 1:
 263             sys.exit(col.Color(col.RED,
 264                                '-w can only be used with a single commit'))
 265
 266     # Read the metadata from the commits. First look at the upstream commit,
 267     # then the ones in the branch. We would like to do something like
 268     # upstream/master~..branch but that isn't possible if upstream/master is
 269     # a merge commit (it will list all the commits that form part of the
 270     # merge)
 271     # Conflicting tags are not a problem for buildman, since it does not use
 272     # them. For example, Series-version is not useful for buildman. On the
 273     # other hand conflicting tags will cause an error. So allow later tags
 274     # to overwrite earlier ones by setting allow_overwrite=True
 275     if options.branch:
 276         if count == -1:
 277             if has_range:
 278                 range_expr = options.branch
 279             else:
 280                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 281                                                       options.branch)
 282             upstream_commit = gitutil.GetUpstream(options.git_dir,
 283                                                   options.branch)
 284             series = patchstream.get_metadata_for_list(upstream_commit,
 285                 options.git_dir, 1, series=None, allow_overwrite=True)
 286
 287             series = patchstream.get_metadata_for_list(range_expr,
 288                     options.git_dir, None, series, allow_overwrite=True)
 289         else:
 290             # Honour the count
 291             series = patchstream.get_metadata_for_list(options.branch,
 292                     options.git_dir, count, series=None, allow_overwrite=True)
 293     else:
 294         series = None
 295         if not options.dry_run:
 296             options.verbose = True
 297             if not options.summary:
 298                 options.show_errors = True
 299
 300     # By default we have one thread per CPU. But if there are not enough jobs
 301     # we can have fewer threads and use a high '-j' value for make.
 302     if options.threads is None:
 303         options.threads = min(multiprocessing.cpu_count(), len(selected))
 304     if not options.jobs:
 305         options.jobs = max(1, (multiprocessing.cpu_count() +
 306                 len(selected) - 1) // len(selected))
 307
 308     if not options.step:
 309         options.step = len(series.commits) - 1
 310
 311     gnu_make = command.Output(os.path.join(options.git,
 312             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 313     if not gnu_make:
 314         sys.exit('GNU Make not found')
 315
 316     # Create a new builder with the selected options.
 317     output_dir = options.output_dir
 318     if options.branch:
 319         dirname = options.branch.replace('/', '_')
 320         # As a special case allow the board directory to be placed in the
 321         # output directory itself rather than any subdirectory.
 322         if not options.no_subdirs:
 323             output_dir = os.path.join(options.output_dir, dirname)
 324         if clean_dir and os.path.exists(output_dir):
 325             shutil.rmtree(output_dir)
 326     builder = Builder(toolchains, output_dir, options.git_dir,
 327             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 328             show_unknown=options.show_unknown, step=options.step,
 329             no_subdirs=options.no_subdirs, full_path=options.full_path,
 330             verbose_build=options.verbose_build,
 331             mrproper=options.mrproper,
 332             per_board_out_dir=options.per_board_out_dir,
 333             config_only=options.config_only,
 334             squash_config_y=not options.preserve_config_y,
 335             warnings_as_errors=options.warnings_as_errors,
 336             work_in_output=options.work_in_output,
 337             test_thread_exceptions=test_thread_exceptions)
 338     builder.force_config_on_failure = not options.quick
 339     if make_func:
 340         builder.do_make = make_func
 341
 342     # For a dry run, just show our actions as a sanity check
 343     if options.dry_run:
 344         ShowActions(series, why_selected, selected, builder, options,
 345                     board_warnings)
 346     else:
 347         builder.force_build = options.force_build
 348         builder.force_build_failures = options.force_build_failures
 349         builder.force_reconfig = options.force_reconfig
 350         builder.in_tree = options.in_tree
 351
 352         # Work out which boards to build
 353         board_selected = boards.GetSelectedDict()
 354
 355         if series:
 356             commits = series.commits
 357             # Number the commits for test purposes
 358             for commit in range(len(commits)):
 359                 commits[commit].sequence = commit
 360         else:
 361             commits = None
 362
 363         Print(GetActionSummary(options.summary, commits, board_selected,
 364                                options))
 365
 366         # We can't show function sizes without board details at present
 367         if options.show_bloat:
 368             options.show_detail = True
 369         builder.SetDisplayOptions(
 370             options.show_errors, options.show_sizes, options.show_detail,
 371             options.show_bloat, options.list_error_boards, options.show_config,
 372             options.show_environment, options.filter_dtb_warnings,
 373             options.filter_migration_warnings)
 374         if options.summary:
 375             builder.ShowSummary(commits, board_selected)
 376         else:
 377             fail, warned, excs = builder.BuildBoards(
 378                 commits, board_selected, options.keep_outputs, options.verbose)
 379             if excs:
 380                 return 102
 381             elif fail:
 382                 return 100
 383             elif warned and not options.ignore_warnings:
 384                 return 101
 385     return 0