tools/buildman/control.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2013 The Chromium OS Authors.
   3 #
   4
   5 import multiprocessing
   6 import os
   7 import shutil
   8 import subprocess
   9 import sys
  10
  11 from buildman import boards
  12 from buildman import bsettings
  13 from buildman import cfgutil
  14 from buildman import toolchain
  15 from buildman.builder import Builder
  16 from patman import command
  17 from patman import gitutil
  18 from patman import patchstream
  19 from patman import terminal
  20 from patman import tools
  21 from patman.terminal import tprint
  22
  23 def GetPlural(count):
  24     """Returns a plural 's' if count is not 1"""
  25     return 's' if count != 1 else ''
  26
  27 def GetActionSummary(is_summary, commits, selected, options):
  28     """Return a string summarising the intended action.
  29
  30     Returns:
  31         Summary string.
  32     """
  33     if commits:
  34         count = len(commits)
  35         count = (count + options.step - 1) // options.step
  36         commit_str = '%d commit%s' % (count, GetPlural(count))
  37     else:
  38         commit_str = 'current source'
  39     str = '%s %s for %d boards' % (
  40         'Summary of' if is_summary else 'Building', commit_str,
  41         len(selected))
  42     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  43             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  44     return str
  45
  46 def ShowActions(series, why_selected, boards_selected, builder, options,
  47                 board_warnings):
  48     """Display a list of actions that we would take, if not a dry run.
  49
  50     Args:
  51         series: Series object
  52         why_selected: Dictionary where each key is a buildman argument
  53                 provided by the user, and the value is the list of boards
  54                 brought in by that argument. For example, 'arm' might bring
  55                 in 400 boards, so in this case the key would be 'arm' and
  56                 the value would be a list of board names.
  57         boards_selected: Dict of selected boards, key is target name,
  58                 value is Board object
  59         builder: The builder that will be used to build the commits
  60         options: Command line options object
  61         board_warnings: List of warnings obtained from board selected
  62     """
  63     col = terminal.Color()
  64     print('Dry run, so not doing much. But I would do this:')
  65     print()
  66     if series:
  67         commits = series.commits
  68     else:
  69         commits = None
  70     print(GetActionSummary(False, commits, boards_selected,
  71             options))
  72     print('Build directory: %s' % builder.base_dir)
  73     if commits:
  74         for upto in range(0, len(series.commits), options.step):
  75             commit = series.commits[upto]
  76             print('   ', col.build(col.YELLOW, commit.hash[:8], bright=False), end=' ')
  77             print(commit.subject)
  78     print()
  79     for arg in why_selected:
  80         if arg != 'all':
  81             print(arg, ': %d boards' % len(why_selected[arg]))
  82             if options.verbose:
  83                 print('   %s' % ' '.join(why_selected[arg]))
  84     print(('Total boards to build for each commit: %d\n' %
  85             len(why_selected['all'])))
  86     if board_warnings:
  87         for warning in board_warnings:
  88             print(col.build(col.YELLOW, warning))
  89
  90 def ShowToolchainPrefix(brds, toolchains):
  91     """Show information about a the tool chain used by one or more boards
  92
  93     The function checks that all boards use the same toolchain, then prints
  94     the correct value for CROSS_COMPILE.
  95
  96     Args:
  97         boards: Boards object containing selected boards
  98         toolchains: Toolchains object containing available toolchains
  99
 100     Return:
 101         None on success, string error message otherwise
 102     """
 103     board_selected = brds.get_selected_dict()
 104     tc_set = set()
 105     for brd in board_selected.values():
 106         tc_set.add(toolchains.Select(brd.arch))
 107     if len(tc_set) != 1:
 108         return 'Supplied boards must share one toolchain'
 109         return False
 110     tc = tc_set.pop()
 111     print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
 112     return None
 113
 114 def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
 115                clean_dir=False, test_thread_exceptions=False):
 116     """The main control code for buildman
 117
 118     Args:
 119         options: Command line options object
 120         args: Command line arguments (list of strings)
 121         toolchains: Toolchains to use - this should be a Toolchains()
 122                 object. If None, then it will be created and scanned
 123         make_func: Make function to use for the builder. This is called
 124                 to execute 'make'. If this is None, the normal function
 125                 will be used, which calls the 'make' tool with suitable
 126                 arguments. This setting is useful for tests.
 127         brds: Boards() object to use, containing a list of available
 128                 boards. If this is None it will be created and scanned.
 129         clean_dir: Used for tests only, indicates that the existing output_dir
 130             should be removed before starting the build
 131         test_thread_exceptions: Uses for tests only, True to make the threads
 132             raise an exception instead of reporting their result. This simulates
 133             a failure in the code somewhere
 134     """
 135     global builder
 136
 137     if options.full_help:
 138         tools.print_full_help(
 139             os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README')
 140         )
 141         return 0
 142
 143     gitutil.setup()
 144     col = terminal.Color()
 145
 146     options.git_dir = os.path.join(options.git, '.git')
 147
 148     no_toolchains = toolchains is None
 149     if no_toolchains:
 150         toolchains = toolchain.Toolchains(options.override_toolchain)
 151
 152     if options.fetch_arch:
 153         if options.fetch_arch == 'list':
 154             sorted_list = toolchains.ListArchs()
 155             print(col.build(col.BLUE, 'Available architectures: %s\n' %
 156                             ' '.join(sorted_list)))
 157             return 0
 158         else:
 159             fetch_arch = options.fetch_arch
 160             if fetch_arch == 'all':
 161                 fetch_arch = ','.join(toolchains.ListArchs())
 162                 print(col.build(col.CYAN, '\nDownloading toolchains: %s' %
 163                                 fetch_arch))
 164             for arch in fetch_arch.split(','):
 165                 print()
 166                 ret = toolchains.FetchAndInstall(arch)
 167                 if ret:
 168                     return ret
 169             return 0
 170
 171     if no_toolchains:
 172         toolchains.GetSettings()
 173         toolchains.Scan(options.list_tool_chains and options.verbose)
 174     if options.list_tool_chains:
 175         toolchains.List()
 176         print()
 177         return 0
 178
 179     if not options.output_dir:
 180         if options.work_in_output:
 181             sys.exit(col.build(col.RED, '-w requires that you specify -o'))
 182         options.output_dir = '..'
 183
 184     # Work out what subset of the boards we are building
 185     if not brds:
 186         if not os.path.exists(options.output_dir):
 187             os.makedirs(options.output_dir)
 188         board_file = os.path.join(options.output_dir, 'boards.cfg')
 189
 190         brds = boards.Boards()
 191         brds.ensure_board_list(board_file,
 192                                options.threads or multiprocessing.cpu_count(),
 193                                force=options.regen_board_list,
 194                                quiet=not options.verbose)
 195         if options.regen_board_list:
 196             return 0
 197         brds.read_boards(board_file)
 198
 199     exclude = []
 200     if options.exclude:
 201         for arg in options.exclude:
 202             exclude += arg.split(',')
 203
 204     if options.boards:
 205         requested_boards = []
 206         for b in options.boards:
 207             requested_boards += b.split(',')
 208     else:
 209         requested_boards = None
 210     why_selected, board_warnings = brds.select_boards(args, exclude,
 211                                                       requested_boards)
 212     selected = brds.get_selected()
 213     if not len(selected):
 214         sys.exit(col.build(col.RED, 'No matching boards found'))
 215
 216     if options.print_prefix:
 217         err = ShowToolchainPrefix(brds, toolchains)
 218         if err:
 219             sys.exit(col.build(col.RED, err))
 220         return 0
 221
 222     # Work out how many commits to build. We want to build everything on the
 223     # branch. We also build the upstream commit as a control so we can see
 224     # problems introduced by the first commit on the branch.
 225     count = options.count
 226     has_range = options.branch and '..' in options.branch
 227     if count == -1:
 228         if not options.branch:
 229             count = 1
 230         else:
 231             if has_range:
 232                 count, msg = gitutil.count_commits_in_range(options.git_dir,
 233                                                          options.branch)
 234             else:
 235                 count, msg = gitutil.count_commits_in_branch(options.git_dir,
 236                                                           options.branch)
 237             if count is None:
 238                 sys.exit(col.build(col.RED, msg))
 239             elif count == 0:
 240                 sys.exit(col.build(col.RED, "Range '%s' has no commits" %
 241                                    options.branch))
 242             if msg:
 243                 print(col.build(col.YELLOW, msg))
 244             count += 1   # Build upstream commit also
 245
 246     if not count:
 247         str = ("No commits found to process in branch '%s': "
 248                "set branch's upstream or use -c flag" % options.branch)
 249         sys.exit(col.build(col.RED, str))
 250     if options.work_in_output:
 251         if len(selected) != 1:
 252             sys.exit(col.build(col.RED,
 253                                '-w can only be used with a single board'))
 254         if count != 1:
 255             sys.exit(col.build(col.RED,
 256                                '-w can only be used with a single commit'))
 257
 258     # Read the metadata from the commits. First look at the upstream commit,
 259     # then the ones in the branch. We would like to do something like
 260     # upstream/master~..branch but that isn't possible if upstream/master is
 261     # a merge commit (it will list all the commits that form part of the
 262     # merge)
 263     # Conflicting tags are not a problem for buildman, since it does not use
 264     # them. For example, Series-version is not useful for buildman. On the
 265     # other hand conflicting tags will cause an error. So allow later tags
 266     # to overwrite earlier ones by setting allow_overwrite=True
 267     if options.branch:
 268         if count == -1:
 269             if has_range:
 270                 range_expr = options.branch
 271             else:
 272                 range_expr = gitutil.get_range_in_branch(options.git_dir,
 273                                                       options.branch)
 274             upstream_commit = gitutil.get_upstream(options.git_dir,
 275                                                   options.branch)
 276             series = patchstream.get_metadata_for_list(upstream_commit,
 277                 options.git_dir, 1, series=None, allow_overwrite=True)
 278
 279             series = patchstream.get_metadata_for_list(range_expr,
 280                     options.git_dir, None, series, allow_overwrite=True)
 281         else:
 282             # Honour the count
 283             series = patchstream.get_metadata_for_list(options.branch,
 284                     options.git_dir, count, series=None, allow_overwrite=True)
 285     else:
 286         series = None
 287         if not options.dry_run:
 288             options.verbose = True
 289             if not options.summary:
 290                 options.show_errors = True
 291
 292     # By default we have one thread per CPU. But if there are not enough jobs
 293     # we can have fewer threads and use a high '-j' value for make.
 294     if options.threads is None:
 295         options.threads = min(multiprocessing.cpu_count(), len(selected))
 296     if not options.jobs:
 297         options.jobs = max(1, (multiprocessing.cpu_count() +
 298                 len(selected) - 1) // len(selected))
 299
 300     if not options.step:
 301         options.step = len(series.commits) - 1
 302
 303     gnu_make = command.output(os.path.join(options.git,
 304             'scripts/show-gnu-make'), raise_on_error=False).rstrip()
 305     if not gnu_make:
 306         sys.exit('GNU Make not found')
 307
 308     # Create a new builder with the selected options.
 309     output_dir = options.output_dir
 310     if options.branch:
 311         dirname = options.branch.replace('/', '_')
 312         # As a special case allow the board directory to be placed in the
 313         # output directory itself rather than any subdirectory.
 314         if not options.no_subdirs:
 315             output_dir = os.path.join(options.output_dir, dirname)
 316         if clean_dir and os.path.exists(output_dir):
 317             shutil.rmtree(output_dir)
 318     adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg)
 319
 320     builder = Builder(toolchains, output_dir, options.git_dir,
 321             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 322             show_unknown=options.show_unknown, step=options.step,
 323             no_subdirs=options.no_subdirs, full_path=options.full_path,
 324             verbose_build=options.verbose_build,
 325             mrproper=options.mrproper,
 326             per_board_out_dir=options.per_board_out_dir,
 327             config_only=options.config_only,
 328             squash_config_y=not options.preserve_config_y,
 329             warnings_as_errors=options.warnings_as_errors,
 330             work_in_output=options.work_in_output,
 331             test_thread_exceptions=test_thread_exceptions,
 332             adjust_cfg=adjust_cfg)
 333     builder.force_config_on_failure = not options.quick
 334     if make_func:
 335         builder.do_make = make_func
 336
 337     # For a dry run, just show our actions as a sanity check
 338     if options.dry_run:
 339         ShowActions(series, why_selected, selected, builder, options,
 340                     board_warnings)
 341     else:
 342         builder.force_build = options.force_build
 343         builder.force_build_failures = options.force_build_failures
 344         builder.force_reconfig = options.force_reconfig
 345         builder.in_tree = options.in_tree
 346
 347         # Work out which boards to build
 348         board_selected = brds.get_selected_dict()
 349
 350         if series:
 351             commits = series.commits
 352             # Number the commits for test purposes
 353             for commit in range(len(commits)):
 354                 commits[commit].sequence = commit
 355         else:
 356             commits = None
 357
 358         if not options.ide:
 359             tprint(GetActionSummary(options.summary, commits, board_selected,
 360                                     options))
 361
 362         # We can't show function sizes without board details at present
 363         if options.show_bloat:
 364             options.show_detail = True
 365         builder.SetDisplayOptions(
 366             options.show_errors, options.show_sizes, options.show_detail,
 367             options.show_bloat, options.list_error_boards, options.show_config,
 368             options.show_environment, options.filter_dtb_warnings,
 369             options.filter_migration_warnings, options.ide)
 370         if options.summary:
 371             builder.ShowSummary(commits, board_selected)
 372         else:
 373             fail, warned, excs = builder.BuildBoards(
 374                 commits, board_selected, options.keep_outputs, options.verbose)
 375             if excs:
 376                 return 102
 377             elif fail:
 378                 return 100
 379             elif warned and not options.ignore_warnings:
 380                 return 101
 381     return 0