1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2013 The Chromium OS Authors.
11 from buildman import board
12 from buildman import bsettings
13 from buildman import toolchain
14 from buildman.builder import Builder
15 from patman import command
16 from patman import gitutil
17 from patman import patchstream
18 from patman import terminal
19 from patman.terminal import Print
22 """Returns a plural 's' if count is not 1"""
23 return 's' if count != 1 else ''
25 def GetActionSummary(is_summary, commits, selected, options):
26 """Return a string summarising the intended action.
33 count = (count + options.step - 1) // options.step
34 commit_str = '%d commit%s' % (count, GetPlural(count))
36 commit_str = 'current source'
37 str = '%s %s for %d boards' % (
38 'Summary of' if is_summary else 'Building', commit_str,
40 str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
41 GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
44 def ShowActions(series, why_selected, boards_selected, builder, options,
46 """Display a list of actions that we would take, if not a dry run.
50 why_selected: Dictionary where each key is a buildman argument
51 provided by the user, and the value is the list of boards
52 brought in by that argument. For example, 'arm' might bring
53 in 400 boards, so in this case the key would be 'arm' and
54 the value would be a list of board names.
55 boards_selected: Dict of selected boards, key is target name,
57 builder: The builder that will be used to build the commits
58 options: Command line options object
59 board_warnings: List of warnings obtained from board selected
61 col = terminal.Color()
62 print('Dry run, so not doing much. But I would do this:')
65 commits = series.commits
68 print(GetActionSummary(False, commits, boards_selected,
70 print('Build directory: %s' % builder.base_dir)
72 for upto in range(0, len(series.commits), options.step):
73 commit = series.commits[upto]
74 print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
77 for arg in why_selected:
79 print(arg, ': %d boards' % len(why_selected[arg]))
81 print(' %s' % ' '.join(why_selected[arg]))
82 print(('Total boards to build for each commit: %d\n' %
83 len(why_selected['all'])))
85 for warning in board_warnings:
86 print(col.Color(col.YELLOW, warning))
88 def ShowToolchainPrefix(boards, toolchains):
89 """Show information about a the tool chain used by one or more boards
91 The function checks that all boards use the same toolchain, then prints
92 the correct value for CROSS_COMPILE.
95 boards: Boards object containing selected boards
96 toolchains: Toolchains object containing available toolchains
99 None on success, string error message otherwise
101 boards = boards.GetSelectedDict()
103 for brd in boards.values():
104 tc_set.add(toolchains.Select(brd.arch))
106 return 'Supplied boards must share one toolchain'
109 print(tc.GetEnvArgs(toolchain.VAR_CROSS_COMPILE))
112 def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
114 """The main control code for buildman
117 options: Command line options object
118 args: Command line arguments (list of strings)
119 toolchains: Toolchains to use - this should be a Toolchains()
120 object. If None, then it will be created and scanned
121 make_func: Make function to use for the builder. This is called
122 to execute 'make'. If this is None, the normal function
123 will be used, which calls the 'make' tool with suitable
124 arguments. This setting is useful for tests.
125 board: Boards() object to use, containing a list of available
126 boards. If this is None it will be created and scanned.
130 if options.full_help:
131 pager = os.getenv('PAGER')
134 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
136 command.Run(pager, fname)
140 col = terminal.Color()
142 options.git_dir = os.path.join(options.git, '.git')
144 no_toolchains = toolchains is None
146 toolchains = toolchain.Toolchains(options.override_toolchain)
148 if options.fetch_arch:
149 if options.fetch_arch == 'list':
150 sorted_list = toolchains.ListArchs()
151 print(col.Color(col.BLUE, 'Available architectures: %s\n' %
152 ' '.join(sorted_list)))
155 fetch_arch = options.fetch_arch
156 if fetch_arch == 'all':
157 fetch_arch = ','.join(toolchains.ListArchs())
158 print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
160 for arch in fetch_arch.split(','):
162 ret = toolchains.FetchAndInstall(arch)
168 toolchains.GetSettings()
169 toolchains.Scan(options.list_tool_chains and options.verbose)
170 if options.list_tool_chains:
175 if options.incremental:
176 print(col.Color(col.RED,
177 'Warning: -I has been removed. See documentation'))
178 if not options.output_dir:
179 if options.work_in_output:
180 sys.exit(col.Color(col.RED, '-w requires that you specify -o'))
181 options.output_dir = '..'
183 # Work out what subset of the boards we are building
185 if not os.path.exists(options.output_dir):
186 os.makedirs(options.output_dir)
187 board_file = os.path.join(options.output_dir, 'boards.cfg')
188 genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py')
189 status = subprocess.call([genboardscfg, '-q', '-o', board_file])
191 sys.exit("Failed to generate boards.cfg")
193 boards = board.Boards()
194 boards.ReadBoards(board_file)
198 for arg in options.exclude:
199 exclude += arg.split(',')
202 requested_boards = []
203 for b in options.boards:
204 requested_boards += b.split(',')
206 requested_boards = None
207 why_selected, board_warnings = boards.SelectBoards(args, exclude,
209 selected = boards.GetSelected()
210 if not len(selected):
211 sys.exit(col.Color(col.RED, 'No matching boards found'))
213 if options.print_prefix:
214 err = ShowToolchainPrefix(boards, toolchains)
216 sys.exit(col.Color(col.RED, err))
219 # Work out how many commits to build. We want to build everything on the
220 # branch. We also build the upstream commit as a control so we can see
221 # problems introduced by the first commit on the branch.
222 count = options.count
223 has_range = options.branch and '..' in options.branch
225 if not options.branch:
229 count, msg = gitutil.CountCommitsInRange(options.git_dir,
232 count, msg = gitutil.CountCommitsInBranch(options.git_dir,
235 sys.exit(col.Color(col.RED, msg))
237 sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
240 print(col.Color(col.YELLOW, msg))
241 count += 1 # Build upstream commit also
244 str = ("No commits found to process in branch '%s': "
245 "set branch's upstream or use -c flag" % options.branch)
246 sys.exit(col.Color(col.RED, str))
247 if options.work_in_output:
248 if len(selected) != 1:
249 sys.exit(col.Color(col.RED,
250 '-w can only be used with a single board'))
252 sys.exit(col.Color(col.RED,
253 '-w can only be used with a single commit'))
255 # Read the metadata from the commits. First look at the upstream commit,
256 # then the ones in the branch. We would like to do something like
257 # upstream/master~..branch but that isn't possible if upstream/master is
258 # a merge commit (it will list all the commits that form part of the
260 # Conflicting tags are not a problem for buildman, since it does not use
261 # them. For example, Series-version is not useful for buildman. On the
262 # other hand conflicting tags will cause an error. So allow later tags
263 # to overwrite earlier ones by setting allow_overwrite=True
267 range_expr = options.branch
269 range_expr = gitutil.GetRangeInBranch(options.git_dir,
271 upstream_commit = gitutil.GetUpstream(options.git_dir,
273 series = patchstream.GetMetaDataForList(upstream_commit,
274 options.git_dir, 1, series=None, allow_overwrite=True)
276 series = patchstream.GetMetaDataForList(range_expr,
277 options.git_dir, None, series, allow_overwrite=True)
280 series = patchstream.GetMetaDataForList(options.branch,
281 options.git_dir, count, series=None, allow_overwrite=True)
284 if not options.dry_run:
285 options.verbose = True
286 if not options.summary:
287 options.show_errors = True
289 # By default we have one thread per CPU. But if there are not enough jobs
290 # we can have fewer threads and use a high '-j' value for make.
291 if not options.threads:
292 options.threads = min(multiprocessing.cpu_count(), len(selected))
294 options.jobs = max(1, (multiprocessing.cpu_count() +
295 len(selected) - 1) // len(selected))
298 options.step = len(series.commits) - 1
300 gnu_make = command.Output(os.path.join(options.git,
301 'scripts/show-gnu-make'), raise_on_error=False).rstrip()
303 sys.exit('GNU Make not found')
305 # Create a new builder with the selected options.
306 output_dir = options.output_dir
308 dirname = options.branch.replace('/', '_')
309 # As a special case allow the board directory to be placed in the
310 # output directory itself rather than any subdirectory.
311 if not options.no_subdirs:
312 output_dir = os.path.join(options.output_dir, dirname)
313 if clean_dir and os.path.exists(output_dir):
314 shutil.rmtree(output_dir)
315 builder = Builder(toolchains, output_dir, options.git_dir,
316 options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
317 show_unknown=options.show_unknown, step=options.step,
318 no_subdirs=options.no_subdirs, full_path=options.full_path,
319 verbose_build=options.verbose_build,
320 mrproper=options.mrproper,
321 per_board_out_dir=options.per_board_out_dir,
322 config_only=options.config_only,
323 squash_config_y=not options.preserve_config_y,
324 warnings_as_errors=options.warnings_as_errors,
325 work_in_output=options.work_in_output)
326 builder.force_config_on_failure = not options.quick
328 builder.do_make = make_func
330 # For a dry run, just show our actions as a sanity check
332 ShowActions(series, why_selected, selected, builder, options,
335 builder.force_build = options.force_build
336 builder.force_build_failures = options.force_build_failures
337 builder.force_reconfig = options.force_reconfig
338 builder.in_tree = options.in_tree
340 # Work out which boards to build
341 board_selected = boards.GetSelectedDict()
344 commits = series.commits
345 # Number the commits for test purposes
346 for commit in range(len(commits)):
347 commits[commit].sequence = commit
351 Print(GetActionSummary(options.summary, commits, board_selected,
354 # We can't show function sizes without board details at present
355 if options.show_bloat:
356 options.show_detail = True
357 builder.SetDisplayOptions(
358 options.show_errors, options.show_sizes, options.show_detail,
359 options.show_bloat, options.list_error_boards, options.show_config,
360 options.show_environment, options.filter_dtb_warnings,
361 options.filter_migration_warnings)
363 builder.ShowSummary(commits, board_selected)
365 fail, warned = builder.BuildBoards(commits, board_selected,
366 options.keep_outputs, options.verbose)
369 elif warned and not options.ignore_warnings: