tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import board
  12 from buildman import bsettings
  13 from buildman import toolchain
  14 from buildman.builder import Builder
  15 from patman import command
  16 from patman import gitutil
  17 from patman import patchstream
  18 from patman import terminal
  19 from patman.terminal import Print
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def ShowToolchainPrefix(boards, toolchains):
  89     """Show information about a the tool chain used by one or more boards
  90
  91     The function checks that all boards use the same toolchain, then prints
  92     the correct value for CROSS_COMPILE.
  93
  94     Args:
  95         boards: Boards object containing selected boards
  96         toolchains: Toolchains object containing available toolchains
  97
  98     Return:
  99         None on success, string error message otherwise
 100     """
 101     boards = boards.GetSelectedDict()
 102     tc_set = set()
 103     for brd in boards.values():
 104         tc_set.add(toolchains.Select(brd.arch))
 105     if len(tc_set) != 1:
 106         return 'Supplied boards must share one toolchain'
 107         return False
 108     tc = tc_set.pop()
 109     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 110     return None
 111
 112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 113                clean_dir=False):
 114     """The main control code for buildman
 115
 116     Args:
 117         options: Command line options object
 118         args: Command line arguments (list of strings)
 119         toolchains: Toolchains to use - this should be a Toolchains()
 120                 object. If None, then it will be created and scanned
 121         make_func: Make function to use for the builder. This is called
 122                 to execute 'make'. If this is None, the normal function
 123                 will be used, which calls the 'make' tool with suitable
 124                 arguments. This setting is useful for tests.
 125         board: Boards() object to use, containing a list of available
 126                 boards. If this is None it will be created and scanned.
 127     """
 128     global builder
 129
 130     if options.full_help:
 131         pager = os.getenv('PAGER')
 132         if not pager:
 133             pager = 'more'
 134         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 135                              'README')
 136         command.Run(pager, fname)
 137         return 0
 138
 139     gitutil.Setup()
 140     col = terminal.Color()
 141
 142     options.git_dir = os.path.join(options.git, '.git')
 143
 144     no_toolchains = toolchains is None
 145     if no_toolchains:
 146         toolchains = toolchain.Toolchains(options.override_toolchain)
 147
 148     if options.fetch_arch:
 149         if options.fetch_arch == 'list':
 150             sorted_list = toolchains.ListArchs()
 151             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 152                             ' '.join(sorted_list)))
 153             return 0
 154         else:
 155             fetch_arch = options.fetch_arch
 156             if fetch_arch == 'all':
 157                 fetch_arch = ','.join(toolchains.ListArchs())
 158                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 159                                 fetch_arch))
 160             for arch in fetch_arch.split(','):
 161                 print()
 162                 ret = toolchains.FetchAndInstall(arch)
 163                 if ret:
 164                     return ret
 165             return 0
 166
 167     if no_toolchains:
 168         toolchains.GetSettings()
 169         toolchains.Scan(options.list_tool_chains and options.verbose)
 170     if options.list_tool_chains:
 171         toolchains.List()
 172         print()
 173         return 0
 174
 175     if options.incremental:
 176         print(col.Color(col.RED,
 177                         'Warning: -I has been removed. See documentation'))
 178     if not options.output_dir:
 179         if options.work_in_output:
 180             sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
 181         options.output_dir = '..'
 182
 183     # Work out what subset of the boards we are building
 184     if not boards:
 185         if not os.path.exists(options.output_dir):
 186             os.makedirs(options.output_dir)
 187         board_file = os.path.join(options.output_dir, 'boards.cfg')
 188         our_path = os.path.dirname(os.path.realpath(__file__))
 189         genboardscfg = os.path.join(our_path, '../genboardscfg.py')
 190         if not os.path.exists(genboardscfg):
 191             genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 192         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 193         if status != 0:
 194             # Older versions don't support -q
 195             status = subprocess.call([genboardscfg, '-o', board_file])
 196             if status != 0:
 197                 sys.exit("Failed to generate boards.cfg")
 198
 199         boards = board.Boards()
 200         boards.ReadBoards(board_file)
 201
 202     exclude = []
 203     if options.exclude:
 204         for arg in options.exclude:
 205             exclude += arg.split(',')
 206
 207     if options.boards:
 208         requested_boards = []
 209         for b in options.boards:
 210             requested_boards += b.split(',')
 211     else:
 212         requested_boards = None
 213     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 214                                                        requested_boards)
 215     selected = boards.GetSelected()
 216     if not len(selected):
 217         sys.exit(col.Color(col.RED, 'No matching boards found'))
 218
 219     if options.print_prefix:
 220         err = ShowToolchainPrefix(boards, toolchains)
 221         if err:
 222             sys.exit(col.Color(col.RED, err))
 223         return 0
 224
 225     # Work out how many commits to build. We want to build everything on the
 226     # branch. We also build the upstream commit as a control so we can see
 227     # problems introduced by the first commit on the branch.
 228     count = options.count
 229     has_range = options.branch and '..' in options.branch
 230     if count == -1:
 231         if not options.branch:
 232             count = 1
 233         else:
 234             if has_range:
 235                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 236                                                          options.branch)
 237             else:
 238                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 239                                                           options.branch)
 240             if count is None:
 241                 sys.exit(col.Color(col.RED, msg))
 242             elif count == 0:
 243                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 244                                    options.branch))
 245             if msg:
 246                 print(col.Color(col.YELLOW, msg))
 247             count += 1   # Build upstream commit also
 248
 249     if not count:
 250         str = ("No commits found to process in branch '%s': "
 251                "set branch's upstream or use -c flag" % options.branch)
 252         sys.exit(col.Color(col.RED, str))
 253     if options.work_in_output:
 254         if len(selected) != 1:
 255             sys.exit(col.Color(col.RED,
 256                                '-w can only be used with a single board'))
 257         if count != 1:
 258             sys.exit(col.Color(col.RED,
 259                                '-w can only be used with a single commit'))
 260
 261     # Read the metadata from the commits. First look at the upstream commit,
 262     # then the ones in the branch. We would like to do something like
 263     # upstream/master~..branch but that isn't possible if upstream/master is
 264     # a merge commit (it will list all the commits that form part of the
 265     # merge)
 266     # Conflicting tags are not a problem for buildman, since it does not use
 267     # them. For example, Series-version is not useful for buildman. On the
 268     # other hand conflicting tags will cause an error. So allow later tags
 269     # to overwrite earlier ones by setting allow_overwrite=True
 270     if options.branch:
 271         if count == -1:
 272             if has_range:
 273                 range_expr = options.branch
 274             else:
 275                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 276                                                       options.branch)
 277             upstream_commit = gitutil.GetUpstream(options.git_dir,
 278                                                   options.branch)
 279             series = patchstream.get_metadata_for_list(upstream_commit,
 280                 options.git_dir, 1, series=None, allow_overwrite=True)
 281
 282             series = patchstream.get_metadata_for_list(range_expr,
 283                     options.git_dir, None, series, allow_overwrite=True)
 284         else:
 285             # Honour the count
 286             series = patchstream.get_metadata_for_list(options.branch,
 287                     options.git_dir, count, series=None, allow_overwrite=True)
 288     else:
 289         series = None
 290         if not options.dry_run:
 291             options.verbose = True
 292             if not options.summary:
 293                 options.show_errors = True
 294
 295     # By default we have one thread per CPU. But if there are not enough jobs
 296     # we can have fewer threads and use a high '-j' value for make.
 297     if options.threads is None:
 298         options.threads = min(multiprocessing.cpu_count(), len(selected))
 299     if not options.jobs:
 300         options.jobs = max(1, (multiprocessing.cpu_count() +
 301                 len(selected) - 1) // len(selected))
 302
 303     if not options.step:
 304         options.step = len(series.commits) - 1
 305
 306     gnu_make = command.Output(os.path.join(options.git,
 307             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 308     if not gnu_make:
 309         sys.exit('GNU Make not found')
 310
 311     # Create a new builder with the selected options.
 312     output_dir = options.output_dir
 313     if options.branch:
 314         dirname = options.branch.replace('/', '_')
 315         # As a special case allow the board directory to be placed in the
 316         # output directory itself rather than any subdirectory.
 317         if not options.no_subdirs:
 318             output_dir = os.path.join(options.output_dir, dirname)
 319         if clean_dir and os.path.exists(output_dir):
 320             shutil.rmtree(output_dir)
 321     builder = Builder(toolchains, output_dir, options.git_dir,
 322             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 323             show_unknown=options.show_unknown, step=options.step,
 324             no_subdirs=options.no_subdirs, full_path=options.full_path,
 325             verbose_build=options.verbose_build,
 326             mrproper=options.mrproper,
 327             per_board_out_dir=options.per_board_out_dir,
 328             config_only=options.config_only,
 329             squash_config_y=not options.preserve_config_y,
 330             warnings_as_errors=options.warnings_as_errors,
 331             work_in_output=options.work_in_output)
 332     builder.force_config_on_failure = not options.quick
 333     if make_func:
 334         builder.do_make = make_func
 335
 336     # For a dry run, just show our actions as a sanity check
 337     if options.dry_run:
 338         ShowActions(series, why_selected, selected, builder, options,
 339                     board_warnings)
 340     else:
 341         builder.force_build = options.force_build
 342         builder.force_build_failures = options.force_build_failures
 343         builder.force_reconfig = options.force_reconfig
 344         builder.in_tree = options.in_tree
 345
 346         # Work out which boards to build
 347         board_selected = boards.GetSelectedDict()
 348
 349         if series:
 350             commits = series.commits
 351             # Number the commits for test purposes
 352             for commit in range(len(commits)):
 353                 commits[commit].sequence = commit
 354         else:
 355             commits = None
 356
 357         Print(GetActionSummary(options.summary, commits, board_selected,
 358                                options))
 359
 360         # We can't show function sizes without board details at present
 361         if options.show_bloat:
 362             options.show_detail = True
 363         builder.SetDisplayOptions(
 364             options.show_errors, options.show_sizes, options.show_detail,
 365             options.show_bloat, options.list_error_boards, options.show_config,
 366             options.show_environment, options.filter_dtb_warnings,
 367             options.filter_migration_warnings)
 368         if options.summary:
 369             builder.ShowSummary(commits, board_selected)
 370         else:
 371             fail, warned = builder.BuildBoards(commits, board_selected,
 372                                 options.keep_outputs, options.verbose)
 373             if fail:
 374                 return 100
 375             elif warned and not options.ignore_warnings:
 376                 return 101
 377     return 0