tools/buildman/builderthread.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import errno
   6 import glob
   7 import os
   8 import shutil
   9 import sys
  10 import threading
  11
  12 import command
  13 import gitutil
  14
  15 RETURN_CODE_RETRY = -1
  16
  17 def Mkdir(dirname, parents = False):
  18     """Make a directory if it doesn't already exist.
  19
  20     Args:
  21         dirname: Directory to create
  22     """
  23     try:
  24         if parents:
  25             os.makedirs(dirname)
  26         else:
  27             os.mkdir(dirname)
  28     except OSError as err:
  29         if err.errno == errno.EEXIST:
  30             if os.path.realpath('.') == os.path.realpath(dirname):
  31                 print "Cannot create the current working directory '%s'!" % dirname
  32                 sys.exit(1)
  33             pass
  34         else:
  35             raise
  36
  37 class BuilderJob:
  38     """Holds information about a job to be performed by a thread
  39
  40     Members:
  41         board: Board object to build
  42         commits: List of commit options to build.
  43     """
  44     def __init__(self):
  45         self.board = None
  46         self.commits = []
  47
  48
  49 class ResultThread(threading.Thread):
  50     """This thread processes results from builder threads.
  51
  52     It simply passes the results on to the builder. There is only one
  53     result thread, and this helps to serialise the build output.
  54     """
  55     def __init__(self, builder):
  56         """Set up a new result thread
  57
  58         Args:
  59             builder: Builder which will be sent each result
  60         """
  61         threading.Thread.__init__(self)
  62         self.builder = builder
  63
  64     def run(self):
  65         """Called to start up the result thread.
  66
  67         We collect the next result job and pass it on to the build.
  68         """
  69         while True:
  70             result = self.builder.out_queue.get()
  71             self.builder.ProcessResult(result)
  72             self.builder.out_queue.task_done()
  73
  74
  75 class BuilderThread(threading.Thread):
  76     """This thread builds U-Boot for a particular board.
  77
  78     An input queue provides each new job. We run 'make' to build U-Boot
  79     and then pass the results on to the output queue.
  80
  81     Members:
  82         builder: The builder which contains information we might need
  83         thread_num: Our thread number (0-n-1), used to decide on a
  84                 temporary directory
  85     """
  86     def __init__(self, builder, thread_num, incremental, per_board_out_dir):
  87         """Set up a new builder thread"""
  88         threading.Thread.__init__(self)
  89         self.builder = builder
  90         self.thread_num = thread_num
  91         self.incremental = incremental
  92         self.per_board_out_dir = per_board_out_dir
  93
  94     def Make(self, commit, brd, stage, cwd, *args, **kwargs):
  95         """Run 'make' on a particular commit and board.
  96
  97         The source code will already be checked out, so the 'commit'
  98         argument is only for information.
  99
 100         Args:
 101             commit: Commit object that is being built
 102             brd: Board object that is being built
 103             stage: Stage of the build. Valid stages are:
 104                         mrproper - can be called to clean source
 105                         config - called to configure for a board
 106                         build - the main make invocation - it does the build
 107             args: A list of arguments to pass to 'make'
 108             kwargs: A list of keyword arguments to pass to command.RunPipe()
 109
 110         Returns:
 111             CommandResult object
 112         """
 113         return self.builder.do_make(commit, brd, stage, cwd, *args,
 114                 **kwargs)
 115
 116     def RunCommit(self, commit_upto, brd, work_dir, do_config, config_only,
 117                   force_build, force_build_failures):
 118         """Build a particular commit.
 119
 120         If the build is already done, and we are not forcing a build, we skip
 121         the build and just return the previously-saved results.
 122
 123         Args:
 124             commit_upto: Commit number to build (0...n-1)
 125             brd: Board object to build
 126             work_dir: Directory to which the source will be checked out
 127             do_config: True to run a make <board>_defconfig on the source
 128             config_only: Only configure the source, do not build it
 129             force_build: Force a build even if one was previously done
 130             force_build_failures: Force a bulid if the previous result showed
 131                 failure
 132
 133         Returns:
 134             tuple containing:
 135                 - CommandResult object containing the results of the build
 136                 - boolean indicating whether 'make config' is still needed
 137         """
 138         # Create a default result - it will be overwritte by the call to
 139         # self.Make() below, in the event that we do a build.
 140         result = command.CommandResult()
 141         result.return_code = 0
 142         if self.builder.in_tree:
 143             out_dir = work_dir
 144         else:
 145             if self.per_board_out_dir:
 146                 out_rel_dir = os.path.join('..', brd.target)
 147             else:
 148                 out_rel_dir = 'build'
 149             out_dir = os.path.join(work_dir, out_rel_dir)
 150
 151         # Check if the job was already completed last time
 152         done_file = self.builder.GetDoneFile(commit_upto, brd.target)
 153         result.already_done = os.path.exists(done_file)
 154         will_build = (force_build or force_build_failures or
 155             not result.already_done)
 156         if result.already_done:
 157             # Get the return code from that build and use it
 158             with open(done_file, 'r') as fd:
 159                 result.return_code = int(fd.readline())
 160
 161             # Check the signal that the build needs to be retried
 162             if result.return_code == RETURN_CODE_RETRY:
 163                 will_build = True
 164             elif will_build:
 165                 err_file = self.builder.GetErrFile(commit_upto, brd.target)
 166                 if os.path.exists(err_file) and os.stat(err_file).st_size:
 167                     result.stderr = 'bad'
 168                 elif not force_build:
 169                     # The build passed, so no need to build it again
 170                     will_build = False
 171
 172         if will_build:
 173             # We are going to have to build it. First, get a toolchain
 174             if not self.toolchain:
 175                 try:
 176                     self.toolchain = self.builder.toolchains.Select(brd.arch)
 177                 except ValueError as err:
 178                     result.return_code = 10
 179                     result.stdout = ''
 180                     result.stderr = str(err)
 181                     # TODO(sjg@chromium.org): This gets swallowed, but needs
 182                     # to be reported.
 183
 184             if self.toolchain:
 185                 # Checkout the right commit
 186                 if self.builder.commits:
 187                     commit = self.builder.commits[commit_upto]
 188                     if self.builder.checkout:
 189                         git_dir = os.path.join(work_dir, '.git')
 190                         gitutil.Checkout(commit.hash, git_dir, work_dir,
 191                                          force=True)
 192                 else:
 193                     commit = 'current'
 194
 195                 # Set up the environment and command line
 196                 env = self.toolchain.MakeEnvironment(self.builder.full_path)
 197                 Mkdir(out_dir)
 198                 args = []
 199                 cwd = work_dir
 200                 src_dir = os.path.realpath(work_dir)
 201                 if not self.builder.in_tree:
 202                     if commit_upto is None:
 203                         # In this case we are building in the original source
 204                         # directory (i.e. the current directory where buildman
 205                         # is invoked. The output directory is set to this
 206                         # thread's selected work directory.
 207                         #
 208                         # Symlinks can confuse U-Boot's Makefile since
 209                         # we may use '..' in our path, so remove them.
 210                         out_dir = os.path.realpath(out_dir)
 211                         args.append('O=%s' % out_dir)
 212                         cwd = None
 213                         src_dir = os.getcwd()
 214                     else:
 215                         args.append('O=%s' % out_rel_dir)
 216                 if self.builder.verbose_build:
 217                     args.append('V=1')
 218                 else:
 219                     args.append('-s')
 220                 if self.builder.num_jobs is not None:
 221                     args.extend(['-j', str(self.builder.num_jobs)])
 222                 if self.builder.warnings_as_errors:
 223                     args.append('KCFLAGS=-Werror')
 224                 config_args = ['%s_defconfig' % brd.target]
 225                 config_out = ''
 226                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
 227
 228                 # If we need to reconfigure, do that now
 229                 if do_config:
 230                     config_out = ''
 231                     if not self.incremental:
 232                         result = self.Make(commit, brd, 'mrproper', cwd,
 233                                 'mrproper', *args, env=env)
 234                         config_out += result.combined
 235                     result = self.Make(commit, brd, 'config', cwd,
 236                             *(args + config_args), env=env)
 237                     config_out += result.combined
 238                     do_config = False   # No need to configure next time
 239                 if result.return_code == 0:
 240                     if config_only:
 241                         args.append('cfg')
 242                     result = self.Make(commit, brd, 'build', cwd, *args,
 243                             env=env)
 244                 result.stderr = result.stderr.replace(src_dir + '/', '')
 245                 if self.builder.verbose_build:
 246                     result.stdout = config_out + result.stdout
 247             else:
 248                 result.return_code = 1
 249                 result.stderr = 'No tool chain for %s\n' % brd.arch
 250             result.already_done = False
 251
 252         result.toolchain = self.toolchain
 253         result.brd = brd
 254         result.commit_upto = commit_upto
 255         result.out_dir = out_dir
 256         return result, do_config
 257
 258     def _WriteResult(self, result, keep_outputs):
 259         """Write a built result to the output directory.
 260
 261         Args:
 262             result: CommandResult object containing result to write
 263             keep_outputs: True to store the output binaries, False
 264                 to delete them
 265         """
 266         # Fatal error
 267         if result.return_code < 0:
 268             return
 269
 270         # If we think this might have been aborted with Ctrl-C, record the
 271         # failure but not that we are 'done' with this board. A retry may fix
 272         # it.
 273         maybe_aborted =  result.stderr and 'No child processes' in result.stderr
 274
 275         if result.already_done:
 276             return
 277
 278         # Write the output and stderr
 279         output_dir = self.builder._GetOutputDir(result.commit_upto)
 280         Mkdir(output_dir)
 281         build_dir = self.builder.GetBuildDir(result.commit_upto,
 282                 result.brd.target)
 283         Mkdir(build_dir)
 284
 285         outfile = os.path.join(build_dir, 'log')
 286         with open(outfile, 'w') as fd:
 287             if result.stdout:
 288                 # We don't want unicode characters in log files
 289                 fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
 290
 291         errfile = self.builder.GetErrFile(result.commit_upto,
 292                 result.brd.target)
 293         if result.stderr:
 294             with open(errfile, 'w') as fd:
 295                 # We don't want unicode characters in log files
 296                 fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
 297         elif os.path.exists(errfile):
 298             os.remove(errfile)
 299
 300         if result.toolchain:
 301             # Write the build result and toolchain information.
 302             done_file = self.builder.GetDoneFile(result.commit_upto,
 303                     result.brd.target)
 304             with open(done_file, 'w') as fd:
 305                 if maybe_aborted:
 306                     # Special code to indicate we need to retry
 307                     fd.write('%s' % RETURN_CODE_RETRY)
 308                 else:
 309                     fd.write('%s' % result.return_code)
 310             with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
 311                 print >>fd, 'gcc', result.toolchain.gcc
 312                 print >>fd, 'path', result.toolchain.path
 313                 print >>fd, 'cross', result.toolchain.cross
 314                 print >>fd, 'arch', result.toolchain.arch
 315                 fd.write('%s' % result.return_code)
 316
 317             # Write out the image and function size information and an objdump
 318             env = result.toolchain.MakeEnvironment(self.builder.full_path)
 319             lines = []
 320             for fname in ['u-boot', 'spl/u-boot-spl']:
 321                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
 322                 nm_result = command.RunPipe([cmd], capture=True,
 323                         capture_stderr=True, cwd=result.out_dir,
 324                         raise_on_error=False, env=env)
 325                 if nm_result.stdout:
 326                     nm = self.builder.GetFuncSizesFile(result.commit_upto,
 327                                     result.brd.target, fname)
 328                     with open(nm, 'w') as fd:
 329                         print >>fd, nm_result.stdout,
 330
 331                 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
 332                 dump_result = command.RunPipe([cmd], capture=True,
 333                         capture_stderr=True, cwd=result.out_dir,
 334                         raise_on_error=False, env=env)
 335                 rodata_size = ''
 336                 if dump_result.stdout:
 337                     objdump = self.builder.GetObjdumpFile(result.commit_upto,
 338                                     result.brd.target, fname)
 339                     with open(objdump, 'w') as fd:
 340                         print >>fd, dump_result.stdout,
 341                     for line in dump_result.stdout.splitlines():
 342                         fields = line.split()
 343                         if len(fields) > 5 and fields[1] == '.rodata':
 344                             rodata_size = fields[2]
 345
 346                 cmd = ['%ssize' % self.toolchain.cross, fname]
 347                 size_result = command.RunPipe([cmd], capture=True,
 348                         capture_stderr=True, cwd=result.out_dir,
 349                         raise_on_error=False, env=env)
 350                 if size_result.stdout:
 351                     lines.append(size_result.stdout.splitlines()[1] + ' ' +
 352                                  rodata_size)
 353
 354             # Extract the environment from U-Boot and dump it out
 355             cmd = ['%sobjcopy' % self.toolchain.cross, '-O', 'binary',
 356                    '-j', '.rodata.default_environment',
 357                    'env/built-in.o', 'uboot.env']
 358             command.RunPipe([cmd], capture=True,
 359                             capture_stderr=True, cwd=result.out_dir,
 360                             raise_on_error=False, env=env)
 361             ubootenv = os.path.join(result.out_dir, 'uboot.env')
 362             self.CopyFiles(result.out_dir, build_dir, '', ['uboot.env'])
 363
 364             # Write out the image sizes file. This is similar to the output
 365             # of binutil's 'size' utility, but it omits the header line and
 366             # adds an additional hex value at the end of each line for the
 367             # rodata size
 368             if len(lines):
 369                 sizes = self.builder.GetSizesFile(result.commit_upto,
 370                                 result.brd.target)
 371                 with open(sizes, 'w') as fd:
 372                     print >>fd, '\n'.join(lines)
 373
 374         # Write out the configuration files, with a special case for SPL
 375         for dirname in ['', 'spl', 'tpl']:
 376             self.CopyFiles(result.out_dir, build_dir, dirname, ['u-boot.cfg',
 377                 'spl/u-boot-spl.cfg', 'tpl/u-boot-tpl.cfg', '.config',
 378                 'include/autoconf.mk', 'include/generated/autoconf.h'])
 379
 380         # Now write the actual build output
 381         if keep_outputs:
 382             self.CopyFiles(result.out_dir, build_dir, '', ['u-boot*', '*.bin',
 383                 '*.map', '*.img', 'MLO', 'SPL', 'include/autoconf.mk',
 384                 'spl/u-boot-spl*'])
 385
 386     def CopyFiles(self, out_dir, build_dir, dirname, patterns):
 387         """Copy files from the build directory to the output.
 388
 389         Args:
 390             out_dir: Path to output directory containing the files
 391             build_dir: Place to copy the files
 392             dirname: Source directory, '' for normal U-Boot, 'spl' for SPL
 393             patterns: A list of filenames (strings) to copy, each relative
 394                to the build directory
 395         """
 396         for pattern in patterns:
 397             file_list = glob.glob(os.path.join(out_dir, dirname, pattern))
 398             for fname in file_list:
 399                 target = os.path.basename(fname)
 400                 if dirname:
 401                     base, ext = os.path.splitext(target)
 402                     if ext:
 403                         target = '%s-%s%s' % (base, dirname, ext)
 404                 shutil.copy(fname, os.path.join(build_dir, target))
 405
 406     def RunJob(self, job):
 407         """Run a single job
 408
 409         A job consists of a building a list of commits for a particular board.
 410
 411         Args:
 412             job: Job to build
 413         """
 414         brd = job.board
 415         work_dir = self.builder.GetThreadDir(self.thread_num)
 416         self.toolchain = None
 417         if job.commits:
 418             # Run 'make board_defconfig' on the first commit
 419             do_config = True
 420             commit_upto  = 0
 421             force_build = False
 422             for commit_upto in range(0, len(job.commits), job.step):
 423                 result, request_config = self.RunCommit(commit_upto, brd,
 424                         work_dir, do_config, self.builder.config_only,
 425                         force_build or self.builder.force_build,
 426                         self.builder.force_build_failures)
 427                 failed = result.return_code or result.stderr
 428                 did_config = do_config
 429                 if failed and not do_config:
 430                     # If our incremental build failed, try building again
 431                     # with a reconfig.
 432                     if self.builder.force_config_on_failure:
 433                         result, request_config = self.RunCommit(commit_upto,
 434                             brd, work_dir, True, False, True, False)
 435                         did_config = True
 436                 if not self.builder.force_reconfig:
 437                     do_config = request_config
 438
 439                 # If we built that commit, then config is done. But if we got
 440                 # an warning, reconfig next time to force it to build the same
 441                 # files that created warnings this time. Otherwise an
 442                 # incremental build may not build the same file, and we will
 443                 # think that the warning has gone away.
 444                 # We could avoid this by using -Werror everywhere...
 445                 # For errors, the problem doesn't happen, since presumably
 446                 # the build stopped and didn't generate output, so will retry
 447                 # that file next time. So we could detect warnings and deal
 448                 # with them specially here. For now, we just reconfigure if
 449                 # anything goes work.
 450                 # Of course this is substantially slower if there are build
 451                 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
 452                 # have problems).
 453                 if (failed and not result.already_done and not did_config and
 454                         self.builder.force_config_on_failure):
 455                     # If this build failed, try the next one with a
 456                     # reconfigure.
 457                     # Sometimes if the board_config.h file changes it can mess
 458                     # with dependencies, and we get:
 459                     # make: *** No rule to make target `include/autoconf.mk',
 460                     #     needed by `depend'.
 461                     do_config = True
 462                     force_build = True
 463                 else:
 464                     force_build = False
 465                     if self.builder.force_config_on_failure:
 466                         if failed:
 467                             do_config = True
 468                     result.commit_upto = commit_upto
 469                     if result.return_code < 0:
 470                         raise ValueError('Interrupt')
 471
 472                 # We have the build results, so output the result
 473                 self._WriteResult(result, job.keep_outputs)
 474                 self.builder.out_queue.put(result)
 475         else:
 476             # Just build the currently checked-out build
 477             result, request_config = self.RunCommit(None, brd, work_dir, True,
 478                         self.builder.config_only, True,
 479                         self.builder.force_build_failures)
 480             result.commit_upto = 0
 481             self._WriteResult(result, job.keep_outputs)
 482             self.builder.out_queue.put(result)
 483
 484     def run(self):
 485         """Our thread's run function
 486
 487         This thread picks a job from the queue, runs it, and then goes to the
 488         next job.
 489         """
 490         while True:
 491             job = self.builder.queue.get()
 492             self.RunJob(job)
 493             self.builder.queue.task_done()