1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2014 Google, Inc
15 RETURN_CODE_RETRY = -1
17 def Mkdir(dirname, parents = False):
18 """Make a directory if it doesn't already exist.
21 dirname: Directory to create
28 except OSError as err:
29 if err.errno == errno.EEXIST:
30 if os.path.realpath('.') == os.path.realpath(dirname):
31 print("Cannot create the current working directory '%s'!" % dirname)
38 """Holds information about a job to be performed by a thread
41 board: Board object to build
42 commits: List of commit options to build.
49 class ResultThread(threading.Thread):
50 """This thread processes results from builder threads.
52 It simply passes the results on to the builder. There is only one
53 result thread, and this helps to serialise the build output.
55 def __init__(self, builder):
56 """Set up a new result thread
59 builder: Builder which will be sent each result
61 threading.Thread.__init__(self)
62 self.builder = builder
65 """Called to start up the result thread.
67 We collect the next result job and pass it on to the build.
70 result = self.builder.out_queue.get()
71 self.builder.ProcessResult(result)
72 self.builder.out_queue.task_done()
75 class BuilderThread(threading.Thread):
76 """This thread builds U-Boot for a particular board.
78 An input queue provides each new job. We run 'make' to build U-Boot
79 and then pass the results on to the output queue.
82 builder: The builder which contains information we might need
83 thread_num: Our thread number (0-n-1), used to decide on a
86 def __init__(self, builder, thread_num, incremental, per_board_out_dir):
87 """Set up a new builder thread"""
88 threading.Thread.__init__(self)
89 self.builder = builder
90 self.thread_num = thread_num
91 self.incremental = incremental
92 self.per_board_out_dir = per_board_out_dir
94 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
95 """Run 'make' on a particular commit and board.
97 The source code will already be checked out, so the 'commit'
98 argument is only for information.
101 commit: Commit object that is being built
102 brd: Board object that is being built
103 stage: Stage of the build. Valid stages are:
104 mrproper - can be called to clean source
105 config - called to configure for a board
106 build - the main make invocation - it does the build
107 args: A list of arguments to pass to 'make'
108 kwargs: A list of keyword arguments to pass to command.RunPipe()
113 return self.builder.do_make(commit, brd, stage, cwd, *args,
116 def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
117 force_build, force_build_failures):
118 """Build a particular commit.
120 If the build is already done, and we are not forcing a build, we skip
121 the build and just return the previously-saved results.
124 commit_upto: Commit number to build (0...n-1)
125 brd: Board object to build
126 work_dir: Directory to which the source will be checked out
127 do_config: True to run a make <board>_defconfig on the source
128 config_only: Only configure the source, do not build it
129 force_build: Force a build even if one was previously done
130 force_build_failures: Force a bulid if the previous result showed
135 - CommandResult object containing the results of the build
136 - boolean indicating whether 'make config' is still needed
138 # Create a default result - it will be overwritte by the call to
139 # self.Make() below, in the event that we do a build.
140 result = command.CommandResult()
141 result.return_code = 0
142 if self.builder.in_tree:
145 if self.per_board_out_dir:
146 out_rel_dir = os.path.join('..', brd.target)
148 out_rel_dir = 'build'
149 out_dir = os.path.join(work_dir, out_rel_dir)
151 # Check if the job was already completed last time
152 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
153 result.already_done = os.path.exists(done_file)
154 will_build = (force_build or force_build_failures or
155 not result.already_done)
156 if result.already_done:
157 # Get the return code from that build and use it
158 with open(done_file, 'r') as fd:
160 result.return_code = int(fd.readline())
162 # The file may be empty due to running out of disk space.
164 result.return_code = RETURN_CODE_RETRY
166 # Check the signal that the build needs to be retried
167 if result.return_code == RETURN_CODE_RETRY:
170 err_file = self.builder.GetErrFile(commit_upto, brd.target)
171 if os.path.exists(err_file) and os.stat(err_file).st_size:
172 result.stderr = 'bad'
173 elif not force_build:
174 # The build passed, so no need to build it again
178 # We are going to have to build it. First, get a toolchain
179 if not self.toolchain:
181 self.toolchain = self.builder.toolchains.Select(brd.arch)
182 except ValueError as err:
183 result.return_code = 10
185 result.stderr = str(err)
186 # TODO(sjg@chromium.org): This gets swallowed, but needs
190 # Checkout the right commit
191 if self.builder.commits:
192 commit = self.builder.commits[commit_upto]
193 if self.builder.checkout:
194 git_dir = os.path.join(work_dir, '.git')
195 gitutil.Checkout(commit.hash, git_dir, work_dir,
200 # Set up the environment and command line
201 env = self.toolchain.MakeEnvironment(self.builder.full_path)
205 src_dir = os.path.realpath(work_dir)
206 if not self.builder.in_tree:
207 if commit_upto is None:
208 # In this case we are building in the original source
209 # directory (i.e. the current directory where buildman
210 # is invoked. The output directory is set to this
211 # thread's selected work directory.
213 # Symlinks can confuse U-Boot's Makefile since
214 # we may use '..' in our path, so remove them.
215 out_dir = os.path.realpath(out_dir)
216 args.append('O=%s' % out_dir)
218 src_dir = os.getcwd()
220 args.append('O=%s' % out_rel_dir)
221 if self.builder.verbose_build:
225 if self.builder.num_jobs is not None:
226 args.extend(['-j', str(self.builder.num_jobs)])
227 if self.builder.warnings_as_errors:
228 args.append('KCFLAGS=-Werror')
229 config_args = ['%s_defconfig' % brd.target]
231 args.extend(self.builder.toolchains.GetMakeArguments(brd))
232 args.extend(self.toolchain.MakeArgs())
234 # If we need to reconfigure, do that now
237 if not self.incremental:
238 result = self.Make(commit, brd, 'mrproper', cwd,
239 'mrproper', *args, env=env)
240 config_out += result.combined
241 result = self.Make(commit, brd, 'config', cwd,
242 *(args + config_args), env=env)
243 config_out += result.combined
244 do_config = False # No need to configure next time
245 if result.return_code == 0:
248 result = self.Make(commit, brd, 'build', cwd, *args,
250 result.stderr = result.stderr.replace(src_dir + '/', '')
251 if self.builder.verbose_build:
252 result.stdout = config_out + result.stdout
254 result.return_code = 1
255 result.stderr = 'No tool chain for %s\n' % brd.arch
256 result.already_done = False
258 result.toolchain = self.toolchain
260 result.commit_upto = commit_upto
261 result.out_dir = out_dir
262 return result, do_config
264 def _WriteResult(self, result, keep_outputs):
265 """Write a built result to the output directory.
268 result: CommandResult object containing result to write
269 keep_outputs: True to store the output binaries, False
273 if result.return_code < 0:
276 # If we think this might have been aborted with Ctrl-C, record the
277 # failure but not that we are 'done' with this board. A retry may fix
279 maybe_aborted = result.stderr and 'No child processes' in result.stderr
281 if result.already_done:
284 # Write the output and stderr
285 output_dir = self.builder._GetOutputDir(result.commit_upto)
287 build_dir = self.builder.GetBuildDir(result.commit_upto,
291 outfile = os.path.join(build_dir, 'log')
292 with open(outfile, 'w') as fd:
294 fd.write(result.stdout)
296 errfile = self.builder.GetErrFile(result.commit_upto,
299 with open(errfile, 'w') as fd:
300 fd.write(result.stderr)
301 elif os.path.exists(errfile):
305 # Write the build result and toolchain information.
306 done_file = self.builder.GetDoneFile(result.commit_upto,
308 with open(done_file, 'w') as fd:
310 # Special code to indicate we need to retry
311 fd.write('%s' % RETURN_CODE_RETRY)
313 fd.write('%s' % result.return_code)
314 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
315 print('gcc', result.toolchain.gcc, file=fd)
316 print('path', result.toolchain.path, file=fd)
317 print('cross', result.toolchain.cross, file=fd)
318 print('arch', result.toolchain.arch, file=fd)
319 fd.write('%s' % result.return_code)
321 # Write out the image and function size information and an objdump
322 env = result.toolchain.MakeEnvironment(self.builder.full_path)
323 with open(os.path.join(build_dir, 'env'), 'w') as fd:
324 for var in sorted(env.keys()):
325 print('%s="%s"' % (var, env[var]), file=fd)
327 for fname in ['u-boot', 'spl/u-boot-spl']:
328 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
329 nm_result = command.RunPipe([cmd], capture=True,
330 capture_stderr=True, cwd=result.out_dir,
331 raise_on_error=False, env=env)
333 nm = self.builder.GetFuncSizesFile(result.commit_upto,
334 result.brd.target, fname)
335 with open(nm, 'w') as fd:
336 print(nm_result.stdout, end=' ', file=fd)
338 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
339 dump_result = command.RunPipe([cmd], capture=True,
340 capture_stderr=True, cwd=result.out_dir,
341 raise_on_error=False, env=env)
343 if dump_result.stdout:
344 objdump = self.builder.GetObjdumpFile(result.commit_upto,
345 result.brd.target, fname)
346 with open(objdump, 'w') as fd:
347 print(dump_result.stdout, end=' ', file=fd)
348 for line in dump_result.stdout.splitlines():
349 fields = line.split()
350 if len(fields) > 5 and fields[1] == '.rodata':
351 rodata_size = fields[2]
353 cmd = ['%ssize' % self.toolchain.cross, fname]
354 size_result = command.RunPipe([cmd], capture=True,
355 capture_stderr=True, cwd=result.out_dir,
356 raise_on_error=False, env=env)
357 if size_result.stdout:
358 lines.append(size_result.stdout.splitlines()[1] + ' ' +
361 # Extract the environment from U-Boot and dump it out
362 cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
363 '-j', '.rodata.default_environment',
364 'env/built-in.o', 'uboot.env']
365 command.RunPipe([cmd], capture=True,
366 capture_stderr=True, cwd=result.out_dir,
367 raise_on_error=False, env=env)
368 ubootenv = os.path.join(result.out_dir, 'uboot.env')
369 self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
371 # Write out the image sizes file. This is similar to the output
372 # of binutil's 'size' utility, but it omits the header line and
373 # adds an additional hex value at the end of each line for the
376 sizes = self.builder.GetSizesFile(result.commit_upto,
378 with open(sizes, 'w') as fd:
379 print('\n'.join(lines), file=fd)
381 # Write out the configuration files, with a special case for SPL
382 for dirname in ['', 'spl', 'tpl']:
383 self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
384 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
385 'include/autoconf.mk', 'include/generated/autoconf.h'])
387 # Now write the actual build output
389 self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
390 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
393 def CopyFiles(self, out_dir, build_dir, dirname, patterns):
394 """Copy files from the build directory to the output.
397 out_dir: Path to output directory containing the files
398 build_dir: Place to copy the files
399 dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
400 patterns: A list of filenames (strings) to copy, each relative
401 to the build directory
403 for pattern in patterns:
404 file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
405 for fname in file_list:
406 target = os.path.basename(fname)
408 base, ext = os.path.splitext(target)
410 target = '%s-%s%s' % (base, dirname, ext)
411 shutil.copy(fname, os.path.join(build_dir, target))
413 def RunJob(self, job):
416 A job consists of a building a list of commits for a particular board.
422 work_dir = self.builder.GetThreadDir(self.thread_num)
423 self.toolchain = None
425 # Run 'make board_defconfig' on the first commit
429 for commit_upto in range(0, len(job.commits), job.step):
430 result, request_config = self.RunCommit(commit_upto, brd,
431 work_dir, do_config, self.builder.config_only,
432 force_build or self.builder.force_build,
433 self.builder.force_build_failures)
434 failed = result.return_code or result.stderr
435 did_config = do_config
436 if failed and not do_config:
437 # If our incremental build failed, try building again
439 if self.builder.force_config_on_failure:
440 result, request_config = self.RunCommit(commit_upto,
441 brd, work_dir, True, False, True, False)
443 if not self.builder.force_reconfig:
444 do_config = request_config
446 # If we built that commit, then config is done. But if we got
447 # an warning, reconfig next time to force it to build the same
448 # files that created warnings this time. Otherwise an
449 # incremental build may not build the same file, and we will
450 # think that the warning has gone away.
451 # We could avoid this by using -Werror everywhere...
452 # For errors, the problem doesn't happen, since presumably
453 # the build stopped and didn't generate output, so will retry
454 # that file next time. So we could detect warnings and deal
455 # with them specially here. For now, we just reconfigure if
456 # anything goes work.
457 # Of course this is substantially slower if there are build
458 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
460 if (failed and not result.already_done and not did_config and
461 self.builder.force_config_on_failure):
462 # If this build failed, try the next one with a
464 # Sometimes if the board_config.h file changes it can mess
465 # with dependencies, and we get:
466 # make: *** No rule to make target `include/autoconf.mk',
467 # needed by `depend'.
472 if self.builder.force_config_on_failure:
475 result.commit_upto = commit_upto
476 if result.return_code < 0:
477 raise ValueError('Interrupt')
479 # We have the build results, so output the result
480 self._WriteResult(result, job.keep_outputs)
481 self.builder.out_queue.put(result)
483 # Just build the currently checked-out build
484 result, request_config = self.RunCommit(None, brd, work_dir, True,
485 self.builder.config_only, True,
486 self.builder.force_build_failures)
487 result.commit_upto = 0
488 self._WriteResult(result, job.keep_outputs)
489 self.builder.out_queue.put(result)
492 """Our thread's run function
494 This thread picks a job from the queue, runs it, and then goes to the
498 job = self.builder.queue.get()
500 self.builder.queue.task_done()