tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import board
  12 from buildman import bsettings
  13 from buildman import toolchain
  14 from buildman.builder import Builder
  15 from patman import command
  16 from patman import gitutil
  17 from patman import patchstream
  18 from patman import terminal
  19 from patman.terminal import Print
  20
  21 def GetPlural(count):
  22     """Returns a plural 's' if count is not 1"""
  23     return 's' if count != 1 else ''
  24
  25 def GetActionSummary(is_summary, commits, selected, options):
  26     """Return a string summarising the intended action.
  27
  28     Returns:
  29         Summary string.
  30     """
  31     if commits:
  32         count = len(commits)
  33         count = (count + options.step - 1) // options.step
  34         commit_str = '%d commit%s' % (count, GetPlural(count))
  35     else:
  36         commit_str = 'current source'
  37     str = '%s %s for %d boards' % (
  38         'Summary of' if is_summary else 'Building', commit_str,
  39         len(selected))
  40     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  41             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  42     return str
  43
  44 def ShowActions(series, why_selected, boards_selected, builder, options,
  45                 board_warnings):
  46     """Display a list of actions that we would take, if not a dry run.
  47
  48     Args:
  49         series: Series object
  50         why_selected: Dictionary where each key is a buildman argument
  51                 provided by the user, and the value is the list of boards
  52                 brought in by that argument. For example, 'arm' might bring
  53                 in 400 boards, so in this case the key would be 'arm' and
  54                 the value would be a list of board names.
  55         boards_selected: Dict of selected boards, key is target name,
  56                 value is Board object
  57         builder: The builder that will be used to build the commits
  58         options: Command line options object
  59         board_warnings: List of warnings obtained from board selected
  60     """
  61     col = terminal.Color()
  62     print('Dry run, so not doing much. But I would do this:')
  63     print()
  64     if series:
  65         commits = series.commits
  66     else:
  67         commits = None
  68     print(GetActionSummary(False, commits, boards_selected,
  69             options))
  70     print('Build directory: %s' % builder.base_dir)
  71     if commits:
  72         for upto in range(0, len(series.commits), options.step):
  73             commit = series.commits[upto]
  74             print('   ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  75             print(commit.subject)
  76     print()
  77     for arg in why_selected:
  78         if arg != 'all':
  79             print(arg, ': %d boards' % len(why_selected[arg]))
  80             if options.verbose:
  81                 print('   %s' % ' '.join(why_selected[arg]))
  82     print(('Total boards to build for each commit: %d\n' %
  83             len(why_selected['all'])))
  84     if board_warnings:
  85         for warning in board_warnings:
  86             print(col.Color(col.YELLOW, warning))
  87
  88 def ShowToolchainPrefix(boards, toolchains):
  89     """Show information about a the tool chain used by one or more boards
  90
  91     The function checks that all boards use the same toolchain, then prints
  92     the correct value for CROSS_COMPILE.
  93
  94     Args:
  95         boards: Boards object containing selected boards
  96         toolchains: Toolchains object containing available toolchains
  97
  98     Return:
  99         None on success, string error message otherwise
 100     """
 101     boards = boards.GetSelectedDict()
 102     tc_set = set()
 103     for brd in boards.values():
 104         tc_set.add(toolchains.Select(brd.arch))
 105     if len(tc_set) != 1:
 106         return 'Supplied boards must share one toolchain'
 107         return False
 108     tc = tc_set.pop()
 109     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 110     return None
 111
 112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
 113                clean_dir=False):
 114     """The main control code for buildman
 115
 116     Args:
 117         options: Command line options object
 118         args: Command line arguments (list of strings)
 119         toolchains: Toolchains to use - this should be a Toolchains()
 120                 object. If None, then it will be created and scanned
 121         make_func: Make function to use for the builder. This is called
 122                 to execute 'make'. If this is None, the normal function
 123                 will be used, which calls the 'make' tool with suitable
 124                 arguments. This setting is useful for tests.
 125         board: Boards() object to use, containing a list of available
 126                 boards. If this is None it will be created and scanned.
 127     """
 128     global builder
 129
 130     if options.full_help:
 131         pager = os.getenv('PAGER')
 132         if not pager:
 133             pager = 'more'
 134         fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
 135                              'README')
 136         command.Run(pager, fname)
 137         return 0
 138
 139     gitutil.Setup()
 140     col = terminal.Color()
 141
 142     options.git_dir = os.path.join(options.git, '.git')
 143
 144     no_toolchains = toolchains is None
 145     if no_toolchains:
 146         toolchains = toolchain.Toolchains(options.override_toolchain)
 147
 148     if options.fetch_arch:
 149         if options.fetch_arch == 'list':
 150             sorted_list = toolchains.ListArchs()
 151             print(col.Color(col.BLUE, 'Available architectures: %s\n' %
 152                             ' '.join(sorted_list)))
 153             return 0
 154         else:
 155             fetch_arch = options.fetch_arch
 156             if fetch_arch == 'all':
 157                 fetch_arch = ','.join(toolchains.ListArchs())
 158                 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
 159                                 fetch_arch))
 160             for arch in fetch_arch.split(','):
 161                 print()
 162                 ret = toolchains.FetchAndInstall(arch)
 163                 if ret:
 164                     return ret
 165             return 0
 166
 167     if no_toolchains:
 168         toolchains.GetSettings()
 169         toolchains.Scan(options.list_tool_chains and options.verbose)
 170     if options.list_tool_chains:
 171         toolchains.List()
 172         print()
 173         return 0
 174
 175     if options.incremental:
 176         print(col.Color(col.RED,
 177                         'Warning: -I has been removed. See documentation'))
 178     if not options.output_dir:
 179         if options.work_in_output:
 180             sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
 181         options.output_dir = '..'
 182
 183     # Work out what subset of the boards we are building
 184     if not boards:
 185         if not os.path.exists(options.output_dir):
 186             os.makedirs(options.output_dir)
 187         board_file = os.path.join(options.output_dir, 'boards.cfg')
 188         genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
 189         status = subprocess.call([genboardscfg, '-q', '-o', board_file])
 190         if status != 0:
 191             sys.exit("Failed to generate boards.cfg")
 192
 193         boards = board.Boards()
 194         boards.ReadBoards(board_file)
 195
 196     exclude = []
 197     if options.exclude:
 198         for arg in options.exclude:
 199             exclude += arg.split(',')
 200
 201     if options.boards:
 202         requested_boards = []
 203         for b in options.boards:
 204             requested_boards += b.split(',')
 205     else:
 206         requested_boards = None
 207     why_selected, board_warnings = boards.SelectBoards(args, exclude,
 208                                                        requested_boards)
 209     selected = boards.GetSelected()
 210     if not len(selected):
 211         sys.exit(col.Color(col.RED, 'No matching boards found'))
 212
 213     if options.print_prefix:
 214         err = ShowToolchainPrefix(boards, toolchains)
 215         if err:
 216             sys.exit(col.Color(col.RED, err))
 217         return 0
 218
 219     # Work out how many commits to build. We want to build everything on the
 220     # branch. We also build the upstream commit as a control so we can see
 221     # problems introduced by the first commit on the branch.
 222     count = options.count
 223     has_range = options.branch and '..' in options.branch
 224     if count == -1:
 225         if not options.branch:
 226             count = 1
 227         else:
 228             if has_range:
 229                 count, msg = gitutil.CountCommitsInRange(options.git_dir,
 230                                                          options.branch)
 231             else:
 232                 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
 233                                                           options.branch)
 234             if count is None:
 235                 sys.exit(col.Color(col.RED, msg))
 236             elif count == 0:
 237                 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
 238                                    options.branch))
 239             if msg:
 240                 print(col.Color(col.YELLOW, msg))
 241             count += 1   # Build upstream commit also
 242
 243     if not count:
 244         str = ("No commits found to process in branch '%s': "
 245                "set branch's upstream or use -c flag" % options.branch)
 246         sys.exit(col.Color(col.RED, str))
 247     if options.work_in_output:
 248         if len(selected) != 1:
 249             sys.exit(col.Color(col.RED,
 250                                '-w can only be used with a single board'))
 251         if count != 1:
 252             sys.exit(col.Color(col.RED,
 253                                '-w can only be used with a single commit'))
 254
 255     # Read the metadata from the commits. First look at the upstream commit,
 256     # then the ones in the branch. We would like to do something like
 257     # upstream/master~..branch but that isn't possible if upstream/master is
 258     # a merge commit (it will list all the commits that form part of the
 259     # merge)
 260     # Conflicting tags are not a problem for buildman, since it does not use
 261     # them. For example, Series-version is not useful for buildman. On the
 262     # other hand conflicting tags will cause an error. So allow later tags
 263     # to overwrite earlier ones by setting allow_overwrite=True
 264     if options.branch:
 265         if count == -1:
 266             if has_range:
 267                 range_expr = options.branch
 268             else:
 269                 range_expr = gitutil.GetRangeInBranch(options.git_dir,
 270                                                       options.branch)
 271             upstream_commit = gitutil.GetUpstream(options.git_dir,
 272                                                   options.branch)
 273             series = patchstream.GetMetaDataForList(upstream_commit,
 274                 options.git_dir, 1, series=None, allow_overwrite=True)
 275
 276             series = patchstream.GetMetaDataForList(range_expr,
 277                     options.git_dir, None, series, allow_overwrite=True)
 278         else:
 279             # Honour the count
 280             series = patchstream.GetMetaDataForList(options.branch,
 281                     options.git_dir, count, series=None, allow_overwrite=True)
 282     else:
 283         series = None
 284         if not options.dry_run:
 285             options.verbose = True
 286             if not options.summary:
 287                 options.show_errors = True
 288
 289     # By default we have one thread per CPU. But if there are not enough jobs
 290     # we can have fewer threads and use a high '-j' value for make.
 291     if not options.threads:
 292         options.threads = min(multiprocessing.cpu_count(), len(selected))
 293     if not options.jobs:
 294         options.jobs = max(1, (multiprocessing.cpu_count() +
 295                 len(selected) - 1) // len(selected))
 296
 297     if not options.step:
 298         options.step = len(series.commits) - 1
 299
 300     gnu_make = command.Output(os.path.join(options.git,
 301             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 302     if not gnu_make:
 303         sys.exit('GNU Make not found')
 304
 305     # Create a new builder with the selected options.
 306     output_dir = options.output_dir
 307     if options.branch:
 308         dirname = options.branch.replace('/', '_')
 309         # As a special case allow the board directory to be placed in the
 310         # output directory itself rather than any subdirectory.
 311         if not options.no_subdirs:
 312             output_dir = os.path.join(options.output_dir, dirname)
 313         if clean_dir and os.path.exists(output_dir):
 314             shutil.rmtree(output_dir)
 315     builder = Builder(toolchains, output_dir, options.git_dir,
 316             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 317             show_unknown=options.show_unknown, step=options.step,
 318             no_subdirs=options.no_subdirs, full_path=options.full_path,
 319             verbose_build=options.verbose_build,
 320             mrproper=options.mrproper,
 321             per_board_out_dir=options.per_board_out_dir,
 322             config_only=options.config_only,
 323             squash_config_y=not options.preserve_config_y,
 324             warnings_as_errors=options.warnings_as_errors,
 325             work_in_output=options.work_in_output)
 326     builder.force_config_on_failure = not options.quick
 327     if make_func:
 328         builder.do_make = make_func
 329
 330     # For a dry run, just show our actions as a sanity check
 331     if options.dry_run:
 332         ShowActions(series, why_selected, selected, builder, options,
 333                     board_warnings)
 334     else:
 335         builder.force_build = options.force_build
 336         builder.force_build_failures = options.force_build_failures
 337         builder.force_reconfig = options.force_reconfig
 338         builder.in_tree = options.in_tree
 339
 340         # Work out which boards to build
 341         board_selected = boards.GetSelectedDict()
 342
 343         if series:
 344             commits = series.commits
 345             # Number the commits for test purposes
 346             for commit in range(len(commits)):
 347                 commits[commit].sequence = commit
 348         else:
 349             commits = None
 350
 351         Print(GetActionSummary(options.summary, commits, board_selected,
 352                                options))
 353
 354         # We can't show function sizes without board details at present
 355         if options.show_bloat:
 356             options.show_detail = True
 357         builder.SetDisplayOptions(
 358             options.show_errors, options.show_sizes, options.show_detail,
 359             options.show_bloat, options.list_error_boards, options.show_config,
 360             options.show_environment, options.filter_dtb_warnings,
 361             options.filter_migration_warnings)
 362         if options.summary:
 363             builder.ShowSummary(commits, board_selected)
 364         else:
 365             fail, warned = builder.BuildBoards(commits, board_selected,
 366                                 options.keep_outputs, options.verbose)
 367             if fail:
 368                 return 100
 369             elif warned and not options.ignore_warnings:
 370                 return 101
 371     return 0