1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2013 The Chromium OS Authors.
11 from buildman import boards
12 from buildman import bsettings
13 from buildman import cfgutil
14 from buildman import toolchain
15 from buildman.builder import Builder
16 from patman import command
17 from patman import gitutil
18 from patman import patchstream
19 from patman import terminal
20 from patman import tools
21 from patman.terminal import tprint
24 """Returns a plural 's' if count is not 1"""
25 return 's' if count != 1 else ''
27 def GetActionSummary(is_summary, commits, selected, options):
28 """Return a string summarising the intended action.
35 count = (count + options.step - 1) // options.step
36 commit_str = '%d commit%s' % (count, GetPlural(count))
38 commit_str = 'current source'
39 str = '%s %s for %d boards' % (
40 'Summary of' if is_summary else 'Building', commit_str,
42 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
43 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
46 def ShowActions(series, why_selected, boards_selected, builder, options,
48 """Display a list of actions that we would take, if not a dry run.
52 why_selected: Dictionary where each key is a buildman argument
53 provided by the user, and the value is the list of boards
54 brought in by that argument. For example, 'arm' might bring
55 in 400 boards, so in this case the key would be 'arm' and
56 the value would be a list of board names.
57 boards_selected: Dict of selected boards, key is target name,
59 builder: The builder that will be used to build the commits
60 options: Command line options object
61 board_warnings: List of warnings obtained from board selected
63 col = terminal.Color()
64 print('Dry run, so not doing much. But I would do this:')
67 commits = series.commits
70 print(GetActionSummary(False, commits, boards_selected,
72 print('Build directory: %s' % builder.base_dir)
74 for upto in range(0, len(series.commits), options.step):
75 commit = series.commits[upto]
76 print(' ', col.build(col.YELLOW, commit.hash[:8], bright=False), end=' ')
79 for arg in why_selected:
81 print(arg, ': %d boards' % len(why_selected[arg]))
83 print(' %s' % ' '.join(why_selected[arg]))
84 print(('Total boards to build for each commit: %d\n' %
85 len(why_selected['all'])))
87 for warning in board_warnings:
88 print(col.build(col.YELLOW, warning))
90 def ShowToolchainPrefix(brds, toolchains):
91 """Show information about a the tool chain used by one or more boards
93 The function checks that all boards use the same toolchain, then prints
94 the correct value for CROSS_COMPILE.
97 boards: Boards object containing selected boards
98 toolchains: Toolchains object containing available toolchains
101 None on success, string error message otherwise
103 board_selected = brds.get_selected_dict()
105 for brd in board_selected.values():
106 tc_set.add(toolchains.Select(brd.arch))
108 return 'Supplied boards must share one toolchain'
111 print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
114 def get_allow_missing(opt_allow, opt_no_allow, num_selected, has_branch):
115 allow_missing = False
116 am_setting = bsettings.GetGlobalItemValue('allow-missing')
118 if am_setting == 'always':
120 if 'multiple' in am_setting and num_selected > 1:
122 if 'branch' in am_setting and has_branch:
128 allow_missing = False
131 def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
132 clean_dir=False, test_thread_exceptions=False):
133 """The main control code for buildman
136 options: Command line options object
137 args: Command line arguments (list of strings)
138 toolchains: Toolchains to use - this should be a Toolchains()
139 object. If None, then it will be created and scanned
140 make_func: Make function to use for the builder. This is called
141 to execute 'make'. If this is None, the normal function
142 will be used, which calls the 'make' tool with suitable
143 arguments. This setting is useful for tests.
144 brds: Boards() object to use, containing a list of available
145 boards. If this is None it will be created and scanned.
146 clean_dir: Used for tests only, indicates that the existing output_dir
147 should be removed before starting the build
148 test_thread_exceptions: Uses for tests only, True to make the threads
149 raise an exception instead of reporting their result. This simulates
150 a failure in the code somewhere
154 if options.full_help:
155 tools.print_full_help(
156 os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
161 col = terminal.Color()
163 options.git_dir = os.path.join(options.git, '.git')
165 no_toolchains = toolchains is None
167 toolchains = toolchain.Toolchains(options.override_toolchain)
169 if options.fetch_arch:
170 if options.fetch_arch == 'list':
171 sorted_list = toolchains.ListArchs()
172 print(col.build(col.BLUE, 'Available architectures: %s\n' %
173 ' '.join(sorted_list)))
176 fetch_arch = options.fetch_arch
177 if fetch_arch == 'all':
178 fetch_arch = ','.join(toolchains.ListArchs())
179 print(col.build(col.CYAN, '\nDownloading toolchains: %s' %
181 for arch in fetch_arch.split(','):
183 ret = toolchains.FetchAndInstall(arch)
189 toolchains.GetSettings()
190 toolchains.Scan(options.list_tool_chains and options.verbose)
191 if options.list_tool_chains:
196 if not options.output_dir:
197 if options.work_in_output:
198 sys.exit(col.build(col.RED, '-w requires that you specify -o'))
199 options.output_dir = '..'
201 # Work out what subset of the boards we are building
203 if not os.path.exists(options.output_dir):
204 os.makedirs(options.output_dir)
205 board_file = os.path.join(options.output_dir, 'boards.cfg')
207 brds = boards.Boards()
208 ok = brds.ensure_board_list(board_file,
209 options.threads or multiprocessing.cpu_count(),
210 force=options.regen_board_list,
211 quiet=not options.verbose)
212 if options.regen_board_list:
213 return 0 if ok else 2
214 brds.read_boards(board_file)
218 for arg in options.exclude:
219 exclude += arg.split(',')
222 requested_boards = []
223 for b in options.boards:
224 requested_boards += b.split(',')
226 requested_boards = None
227 why_selected, board_warnings = brds.select_boards(args, exclude,
229 selected = brds.get_selected()
230 if not len(selected):
231 sys.exit(col.build(col.RED, 'No matching boards found'))
233 if options.print_prefix:
234 err = ShowToolchainPrefix(brds, toolchains)
236 sys.exit(col.build(col.RED, err))
239 # Work out how many commits to build. We want to build everything on the
240 # branch. We also build the upstream commit as a control so we can see
241 # problems introduced by the first commit on the branch.
242 count = options.count
243 has_range = options.branch and '..' in options.branch
245 if not options.branch:
249 count, msg = gitutil.count_commits_in_range(options.git_dir,
252 count, msg = gitutil.count_commits_in_branch(options.git_dir,
255 sys.exit(col.build(col.RED, msg))
257 sys.exit(col.build(col.RED, "Range '%s' has no commits" %
260 print(col.build(col.YELLOW, msg))
261 count += 1 # Build upstream commit also
264 str = ("No commits found to process in branch '%s': "
265 "set branch's upstream or use -c flag" % options.branch)
266 sys.exit(col.build(col.RED, str))
267 if options.work_in_output:
268 if len(selected) != 1:
269 sys.exit(col.build(col.RED,
270 '-w can only be used with a single board'))
272 sys.exit(col.build(col.RED,
273 '-w can only be used with a single commit'))
275 # Read the metadata from the commits. First look at the upstream commit,
276 # then the ones in the branch. We would like to do something like
277 # upstream/master~..branch but that isn't possible if upstream/master is
278 # a merge commit (it will list all the commits that form part of the
280 # Conflicting tags are not a problem for buildman, since it does not use
281 # them. For example, Series-version is not useful for buildman. On the
282 # other hand conflicting tags will cause an error. So allow later tags
283 # to overwrite earlier ones by setting allow_overwrite=True
287 range_expr = options.branch
289 range_expr = gitutil.get_range_in_branch(options.git_dir,
291 upstream_commit = gitutil.get_upstream(options.git_dir,
293 series = patchstream.get_metadata_for_list(upstream_commit,
294 options.git_dir, 1, series=None, allow_overwrite=True)
296 series = patchstream.get_metadata_for_list(range_expr,
297 options.git_dir, None, series, allow_overwrite=True)
300 series = patchstream.get_metadata_for_list(options.branch,
301 options.git_dir, count, series=None, allow_overwrite=True)
304 if not options.dry_run:
305 options.verbose = True
306 if not options.summary:
307 options.show_errors = True
309 # By default we have one thread per CPU. But if there are not enough jobs
310 # we can have fewer threads and use a high '-j' value for make.
311 if options.threads is None:
312 options.threads = min(multiprocessing.cpu_count(), len(selected))
314 options.jobs = max(1, (multiprocessing.cpu_count() +
315 len(selected) - 1) // len(selected))
318 options.step = len(series.commits) - 1
320 gnu_make = command.output(os.path.join(options.git,
321 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
323 sys.exit('GNU Make not found')
325 allow_missing = get_allow_missing(options.allow_missing,
326 options.no_allow_missing, len(selected),
329 # Create a new builder with the selected options.
330 output_dir = options.output_dir
332 dirname = options.branch.replace('/', '_')
333 # As a special case allow the board directory to be placed in the
334 # output directory itself rather than any subdirectory.
335 if not options.no_subdirs:
336 output_dir = os.path.join(options.output_dir, dirname)
337 if clean_dir and os.path.exists(output_dir):
338 shutil.rmtree(output_dir)
339 adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg)
341 # Drop LOCALVERSION_AUTO since it changes the version string on every commit
342 if options.reproducible_builds:
343 # If these are mentioned, leave the local version alone
344 if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg:
345 print('Not dropping LOCALVERSION_AUTO for reproducible build')
347 adjust_cfg['LOCALVERSION_AUTO'] = '~'
349 builder = Builder(toolchains, output_dir, options.git_dir,
350 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
351 show_unknown=options.show_unknown, step=options.step,
352 no_subdirs=options.no_subdirs, full_path=options.full_path,
353 verbose_build=options.verbose_build,
354 mrproper=options.mrproper,
355 per_board_out_dir=options.per_board_out_dir,
356 config_only=options.config_only,
357 squash_config_y=not options.preserve_config_y,
358 warnings_as_errors=options.warnings_as_errors,
359 work_in_output=options.work_in_output,
360 test_thread_exceptions=test_thread_exceptions,
361 adjust_cfg=adjust_cfg,
362 allow_missing=allow_missing, no_lto=options.no_lto,
363 reproducible_builds=options.reproducible_builds)
364 builder.force_config_on_failure = not options.quick
366 builder.do_make = make_func
368 # For a dry run, just show our actions as a sanity check
370 ShowActions(series, why_selected, selected, builder, options,
373 builder.force_build = options.force_build
374 builder.force_build_failures = options.force_build_failures
375 builder.force_reconfig = options.force_reconfig
376 builder.in_tree = options.in_tree
378 # Work out which boards to build
379 board_selected = brds.get_selected_dict()
382 commits = series.commits
383 # Number the commits for test purposes
384 for commit in range(len(commits)):
385 commits[commit].sequence = commit
390 tprint(GetActionSummary(options.summary, commits, board_selected,
393 # We can't show function sizes without board details at present
394 if options.show_bloat:
395 options.show_detail = True
396 builder.SetDisplayOptions(
397 options.show_errors, options.show_sizes, options.show_detail,
398 options.show_bloat, options.list_error_boards, options.show_config,
399 options.show_environment, options.filter_dtb_warnings,
400 options.filter_migration_warnings, options.ide)
402 builder.ShowSummary(commits, board_selected)
404 fail, warned, excs = builder.BuildBoards(
405 commits, board_selected, options.keep_outputs, options.verbose)
410 elif warned and not options.ignore_warnings: