tools/buildman/func_test.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import os
   6 import shutil
   7 import sys
   8 import tempfile
   9 import unittest
  10
  11 import board
  12 import bsettings
  13 import cmdline
  14 import command
  15 import control
  16 import gitutil
  17 import terminal
  18 import toolchain
  19
  20 settings_data = '''
  21 # Buildman settings file
  22
  23 [toolchain]
  24
  25 [toolchain-alias]
  26
  27 [make-flags]
  28 src=/home/sjg/c/src
  29 chroot=/home/sjg/c/chroot
  30 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  31 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  32 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  33 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  34 '''
  35
  36 boards = [
  37     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  38     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  39     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  40     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  41 ]
  42
  43 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  44 39403bb patman: Use --no-pager' to stop git from forking a pager
  45 db6e6f2 patman: Remove the -a option
  46 f2ccf03 patman: Correct unit tests to run correctly
  47 1d097f9 patman: Fix indentation in terminal.py
  48 d073747 patman: Support the 'reverse' option for 'git log
  49 """
  50
  51 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  52 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  53 Date:   Fri Aug 22 19:12:41 2014 +0900
  54
  55     buildman: refactor help message
  56
  57     "buildman [options]" is displayed by default.
  58
  59     Append the rest of help messages to parser.usage
  60     instead of replacing it.
  61
  62     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  63     Drop it from the usage.
  64
  65     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  66 """,
  67 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  68 Author: Simon Glass <sjg@chromium.org>
  69 Date:   Thu Aug 14 16:48:25 2014 -0600
  70
  71     patman: Support the 'reverse' option for 'git log'
  72
  73     This option is currently not supported, but needs to be, for buildman to
  74     operate as expected.
  75
  76     Series-changes: 7
  77     - Add new patch to fix the 'reverse' bug
  78
  79     Series-version: 8
  80
  81     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  82     Reported-by: York Sun <yorksun@freescale.com>
  83     Signed-off-by: Simon Glass <sjg@chromium.org>
  84
  85 """,
  86 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  87 Author: Simon Glass <sjg@chromium.org>
  88 Date:   Sat Aug 9 11:44:32 2014 -0600
  89
  90     patman: Fix indentation in terminal.py
  91
  92     This code came from a different project with 2-character indentation. Fix
  93     it for U-Boot.
  94
  95     Series-changes: 6
  96     - Add new patch to fix indentation in teminal.py
  97
  98     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
  99     Signed-off-by: Simon Glass <sjg@chromium.org>
 100
 101 """,
 102 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 103 Author: Simon Glass <sjg@chromium.org>
 104 Date:   Sat Aug 9 11:08:24 2014 -0600
 105
 106     patman: Correct unit tests to run correctly
 107
 108     It seems that doctest behaves differently now, and some of the unit tests
 109     do not run. Adjust the tests to work correctly.
 110
 111      ./tools/patman/patman --test
 112     <unittest.result.TestResult run=10 errors=0 failures=0>
 113
 114     Series-changes: 6
 115     - Add new patch to fix patman unit tests
 116
 117     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 118
 119 """,
 120 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 121 Author: Simon Glass <sjg@chromium.org>
 122 Date:   Sat Aug 9 12:06:02 2014 -0600
 123
 124     patman: Remove the -a option
 125
 126     It seems that this is no longer needed, since checkpatch.pl will catch
 127     whitespace problems in patches. Also the option is not widely used, so
 128     it seems safe to just remove it.
 129
 130     Series-changes: 6
 131     - Add new patch to remove patman's -a option
 132
 133     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 134     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 135
 136 """,
 137 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 138 Author: Simon Glass <sjg@chromium.org>
 139 Date:   Thu Aug 14 21:50:52 2014 -0600
 140
 141     patman: Use --no-pager' to stop git from forking a pager
 142
 143 """,
 144 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 145 Author: Simon Glass <sjg@chromium.org>
 146 Date:   Fri Aug 22 15:57:39 2014 -0600
 147
 148     patman: Avoid changing the order of tags
 149
 150     patman collects tags that it sees in the commit and places them nicely
 151     sorted at the end of the patch. However, this is not really necessary and
 152     in fact is apparently not desirable.
 153
 154     Series-changes: 9
 155     - Add new patch to avoid changing the order of tags
 156
 157     Series-version: 9
 158
 159     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 160     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 161 """]
 162
 163 TEST_BRANCH = '__testbranch'
 164
 165 class TestFunctional(unittest.TestCase):
 166     """Functional test for buildman.
 167
 168     This aims to test from just below the invocation of buildman (parsing
 169     of arguments) to 'make' and 'git' invocation. It is not a true
 170     emd-to-end test, as it mocks git, make and the tool chain. But this
 171     makes it easier to detect when the builder is doing the wrong thing,
 172     since in many cases this test code will fail. For example, only a
 173     very limited subset of 'git' arguments is supported - anything
 174     unexpected will fail.
 175     """
 176     def setUp(self):
 177         self._base_dir = tempfile.mkdtemp()
 178         self._git_dir = os.path.join(self._base_dir, 'src')
 179         self._buildman_pathname = sys.argv[0]
 180         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 181         command.test_result = self._HandleCommand
 182         self.setupToolchains()
 183         self._toolchains.Add('arm-gcc', test=False)
 184         self._toolchains.Add('powerpc-gcc', test=False)
 185         bsettings.Setup(None)
 186         bsettings.AddFile(settings_data)
 187         self._boards = board.Boards()
 188         for brd in boards:
 189             self._boards.AddBoard(board.Board(*brd))
 190
 191         # Directories where the source been cloned
 192         self._clone_dirs = []
 193         self._commits = len(commit_shortlog.splitlines()) + 1
 194         self._total_builds = self._commits * len(boards)
 195
 196         # Number of calls to make
 197         self._make_calls = 0
 198
 199         # Map of [board, commit] to error messages
 200         self._error = {}
 201
 202         self._test_branch = TEST_BRANCH
 203
 204         # Avoid sending any output and clear all terminal output
 205         terminal.SetPrintTestMode()
 206         terminal.GetPrintTestLines()
 207
 208     def tearDown(self):
 209         shutil.rmtree(self._base_dir)
 210
 211     def setupToolchains(self):
 212         self._toolchains = toolchain.Toolchains()
 213         self._toolchains.Add('gcc', test=False)
 214
 215     def _RunBuildman(self, *args):
 216         return command.RunPipe([[self._buildman_pathname] + list(args)],
 217                 capture=True, capture_stderr=True)
 218
 219     def _RunControl(self, *args, **kwargs):
 220         sys.argv = [sys.argv[0]] + list(args)
 221         options, args = cmdline.ParseArgs()
 222         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 223                 make_func=self._HandleMake, boards=self._boards,
 224                 clean_dir=kwargs.get('clean_dir', True))
 225         self._builder = control.builder
 226         return result
 227
 228     def testFullHelp(self):
 229         command.test_result = None
 230         result = self._RunBuildman('-H')
 231         help_file = os.path.join(self._buildman_dir, 'README')
 232         # Remove possible extraneous strings
 233         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 234         gothelp = result.stdout.replace(extra, '')
 235         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 236         self.assertEqual(0, len(result.stderr))
 237         self.assertEqual(0, result.return_code)
 238
 239     def testHelp(self):
 240         command.test_result = None
 241         result = self._RunBuildman('-h')
 242         help_file = os.path.join(self._buildman_dir, 'README')
 243         self.assertTrue(len(result.stdout) > 1000)
 244         self.assertEqual(0, len(result.stderr))
 245         self.assertEqual(0, result.return_code)
 246
 247     def testGitSetup(self):
 248         """Test gitutils.Setup(), from outside the module itself"""
 249         command.test_result = command.CommandResult(return_code=1)
 250         gitutil.Setup()
 251         self.assertEqual(gitutil.use_no_decorate, False)
 252
 253         command.test_result = command.CommandResult(return_code=0)
 254         gitutil.Setup()
 255         self.assertEqual(gitutil.use_no_decorate, True)
 256
 257     def _HandleCommandGitLog(self, args):
 258         if args[-1] == '--':
 259             args = args[:-1]
 260         if '-n0' in args:
 261             return command.CommandResult(return_code=0)
 262         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 263             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 264         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 265             if args[-1] == self._test_branch:
 266                 count = int(args[3][2:])
 267                 return command.CommandResult(return_code=0,
 268                                             stdout=''.join(commit_log[:count]))
 269
 270         # Not handled, so abort
 271         print 'git log', args
 272         sys.exit(1)
 273
 274     def _HandleCommandGitConfig(self, args):
 275         config = args[0]
 276         if config == 'sendemail.aliasesfile':
 277             return command.CommandResult(return_code=0)
 278         elif config.startswith('branch.badbranch'):
 279             return command.CommandResult(return_code=1)
 280         elif config == 'branch.%s.remote' % self._test_branch:
 281             return command.CommandResult(return_code=0, stdout='upstream\n')
 282         elif config == 'branch.%s.merge' % self._test_branch:
 283             return command.CommandResult(return_code=0,
 284                                          stdout='refs/heads/master\n')
 285
 286         # Not handled, so abort
 287         print 'git config', args
 288         sys.exit(1)
 289
 290     def _HandleCommandGit(self, in_args):
 291         """Handle execution of a git command
 292
 293         This uses a hacked-up parser.
 294
 295         Args:
 296             in_args: Arguments after 'git' from the command line
 297         """
 298         git_args = []           # Top-level arguments to git itself
 299         sub_cmd = None          # Git sub-command selected
 300         args = []               # Arguments to the git sub-command
 301         for arg in in_args:
 302             if sub_cmd:
 303                 args.append(arg)
 304             elif arg[0] == '-':
 305                 git_args.append(arg)
 306             else:
 307                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 308                     git_args.append(arg)
 309                 else:
 310                     sub_cmd = arg
 311         if sub_cmd == 'config':
 312             return self._HandleCommandGitConfig(args)
 313         elif sub_cmd == 'log':
 314             return self._HandleCommandGitLog(args)
 315         elif sub_cmd == 'clone':
 316             return command.CommandResult(return_code=0)
 317         elif sub_cmd == 'checkout':
 318             return command.CommandResult(return_code=0)
 319
 320         # Not handled, so abort
 321         print 'git', git_args, sub_cmd, args
 322         sys.exit(1)
 323
 324     def _HandleCommandNm(self, args):
 325         return command.CommandResult(return_code=0)
 326
 327     def _HandleCommandObjdump(self, args):
 328         return command.CommandResult(return_code=0)
 329
 330     def _HandleCommandSize(self, args):
 331         return command.CommandResult(return_code=0)
 332
 333     def _HandleCommand(self, **kwargs):
 334         """Handle a command execution.
 335
 336         The command is in kwargs['pipe-list'], as a list of pipes, each a
 337         list of commands. The command should be emulated as required for
 338         testing purposes.
 339
 340         Returns:
 341             A CommandResult object
 342         """
 343         pipe_list = kwargs['pipe_list']
 344         wc = False
 345         if len(pipe_list) != 1:
 346             if pipe_list[1] == ['wc', '-l']:
 347                 wc = True
 348             else:
 349                 print 'invalid pipe', kwargs
 350                 sys.exit(1)
 351         cmd = pipe_list[0][0]
 352         args = pipe_list[0][1:]
 353         result = None
 354         if cmd == 'git':
 355             result = self._HandleCommandGit(args)
 356         elif cmd == './scripts/show-gnu-make':
 357             return command.CommandResult(return_code=0, stdout='make')
 358         elif cmd.endswith('nm'):
 359             return self._HandleCommandNm(args)
 360         elif cmd.endswith('objdump'):
 361             return self._HandleCommandObjdump(args)
 362         elif cmd.endswith( 'size'):
 363             return self._HandleCommandSize(args)
 364
 365         if not result:
 366             # Not handled, so abort
 367             print 'unknown command', kwargs
 368             sys.exit(1)
 369
 370         if wc:
 371             result.stdout = len(result.stdout.splitlines())
 372         return result
 373
 374     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 375         """Handle execution of 'make'
 376
 377         Args:
 378             commit: Commit object that is being built
 379             brd: Board object that is being built
 380             stage: Stage that we are at (mrproper, config, build)
 381             cwd: Directory where make should be run
 382             args: Arguments to pass to make
 383             kwargs: Arguments to pass to command.RunPipe()
 384         """
 385         self._make_calls += 1
 386         if stage == 'mrproper':
 387             return command.CommandResult(return_code=0)
 388         elif stage == 'config':
 389             return command.CommandResult(return_code=0,
 390                     combined='Test configuration complete')
 391         elif stage == 'build':
 392             stderr = ''
 393             if type(commit) is not str:
 394                 stderr = self._error.get((brd.target, commit.sequence))
 395             if stderr:
 396                 return command.CommandResult(return_code=1, stderr=stderr)
 397             return command.CommandResult(return_code=0)
 398
 399         # Not handled, so abort
 400         print 'make', stage
 401         sys.exit(1)
 402
 403     # Example function to print output lines
 404     def print_lines(self, lines):
 405         print len(lines)
 406         for line in lines:
 407             print line
 408         #self.print_lines(terminal.GetPrintTestLines())
 409
 410     def testNoBoards(self):
 411         """Test that buildman aborts when there are no boards"""
 412         self._boards = board.Boards()
 413         with self.assertRaises(SystemExit):
 414             self._RunControl()
 415
 416     def testCurrentSource(self):
 417         """Very simple test to invoke buildman on the current source"""
 418         self.setupToolchains();
 419         self._RunControl()
 420         lines = terminal.GetPrintTestLines()
 421         self.assertIn('Building current source for %d boards' % len(boards),
 422                       lines[0].text)
 423
 424     def testBadBranch(self):
 425         """Test that we can detect an invalid branch"""
 426         with self.assertRaises(ValueError):
 427             self._RunControl('-b', 'badbranch')
 428
 429     def testBadToolchain(self):
 430         """Test that missing toolchains are detected"""
 431         self.setupToolchains();
 432         ret_code = self._RunControl('-b', TEST_BRANCH)
 433         lines = terminal.GetPrintTestLines()
 434
 435         # Buildman always builds the upstream commit as well
 436         self.assertIn('Building %d commits for %d boards' %
 437                 (self._commits, len(boards)), lines[0].text)
 438         self.assertEqual(self._builder.count, self._total_builds)
 439
 440         # Only sandbox should succeed, the others don't have toolchains
 441         self.assertEqual(self._builder.fail,
 442                          self._total_builds - self._commits)
 443         self.assertEqual(ret_code, 128)
 444
 445         for commit in range(self._commits):
 446             for board in self._boards.GetList():
 447                 if board.arch != 'sandbox':
 448                   errfile = self._builder.GetErrFile(commit, board.target)
 449                   fd = open(errfile)
 450                   self.assertEqual(fd.readlines(),
 451                           ['No tool chain for %s\n' % board.arch])
 452                   fd.close()
 453
 454     def testBranch(self):
 455         """Test building a branch with all toolchains present"""
 456         self._RunControl('-b', TEST_BRANCH)
 457         self.assertEqual(self._builder.count, self._total_builds)
 458         self.assertEqual(self._builder.fail, 0)
 459
 460     def testCount(self):
 461         """Test building a specific number of commitst"""
 462         self._RunControl('-b', TEST_BRANCH, '-c2')
 463         self.assertEqual(self._builder.count, 2 * len(boards))
 464         self.assertEqual(self._builder.fail, 0)
 465         # Each board has a mrproper, config, and then one make per commit
 466         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 467
 468     def testIncremental(self):
 469         """Test building a branch twice - the second time should do nothing"""
 470         self._RunControl('-b', TEST_BRANCH)
 471
 472         # Each board has a mrproper, config, and then one make per commit
 473         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 474         self._make_calls = 0
 475         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 476         self.assertEqual(self._make_calls, 0)
 477         self.assertEqual(self._builder.count, self._total_builds)
 478         self.assertEqual(self._builder.fail, 0)
 479
 480     def testForceBuild(self):
 481         """The -f flag should force a rebuild"""
 482         self._RunControl('-b', TEST_BRANCH)
 483         self._make_calls = 0
 484         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 485         # Each board has a mrproper, config, and then one make per commit
 486         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 487
 488     def testForceReconfigure(self):
 489         """The -f flag should force a rebuild"""
 490         self._RunControl('-b', TEST_BRANCH, '-C')
 491         # Each commit has a mrproper, config and make
 492         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 493
 494     def testErrors(self):
 495         """Test handling of build errors"""
 496         self._error['board2', 1] = 'fred\n'
 497         self._RunControl('-b', TEST_BRANCH)
 498         self.assertEqual(self._builder.count, self._total_builds)
 499         self.assertEqual(self._builder.fail, 1)
 500
 501         # Remove the error. This should have no effect since the commit will
 502         # not be rebuilt
 503         del self._error['board2', 1]
 504         self._make_calls = 0
 505         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 506         self.assertEqual(self._builder.count, self._total_builds)
 507         self.assertEqual(self._make_calls, 0)
 508         self.assertEqual(self._builder.fail, 1)
 509
 510         # Now use the -F flag to force rebuild of the bad commit
 511         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 512         self.assertEqual(self._builder.count, self._total_builds)
 513         self.assertEqual(self._builder.fail, 0)
 514         self.assertEqual(self._make_calls, 3)
 515
 516     def testBranchWithSlash(self):
 517         """Test building a branch with a '/' in the name"""
 518         self._test_branch = '/__dev/__testbranch'
 519         self._RunControl('-b', self._test_branch, clean_dir=False)
 520         self.assertEqual(self._builder.count, self._total_builds)
 521         self.assertEqual(self._builder.fail, 0)