tools/buildman/func_test.py

   1 #
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4 # SPDX-License-Identifier:      GPL-2.0+
   5 #
   6
   7 import os
   8 import shutil
   9 import sys
  10 import tempfile
  11 import unittest
  12
  13 import board
  14 import bsettings
  15 import cmdline
  16 import command
  17 import control
  18 import gitutil
  19 import terminal
  20 import toolchain
  21
  22 settings_data = '''
  23 # Buildman settings file
  24
  25 [toolchain]
  26
  27 [toolchain-alias]
  28
  29 [make-flags]
  30 src=/home/sjg/c/src
  31 chroot=/home/sjg/c/chroot
  32 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  33 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  34 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  35 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  36 '''
  37
  38 boards = [
  39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  40     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  41     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  42     ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
  43     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  44 ]
  45
  46 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  47 39403bb patman: Use --no-pager' to stop git from forking a pager
  48 db6e6f2 patman: Remove the -a option
  49 f2ccf03 patman: Correct unit tests to run correctly
  50 1d097f9 patman: Fix indentation in terminal.py
  51 d073747 patman: Support the 'reverse' option for 'git log
  52 """
  53
  54 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  55 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  56 Date:   Fri Aug 22 19:12:41 2014 +0900
  57
  58     buildman: refactor help message
  59
  60     "buildman [options]" is displayed by default.
  61
  62     Append the rest of help messages to parser.usage
  63     instead of replacing it.
  64
  65     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  66     Drop it from the usage.
  67
  68     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  69 """,
  70 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  71 Author: Simon Glass <sjg@chromium.org>
  72 Date:   Thu Aug 14 16:48:25 2014 -0600
  73
  74     patman: Support the 'reverse' option for 'git log'
  75
  76     This option is currently not supported, but needs to be, for buildman to
  77     operate as expected.
  78
  79     Series-changes: 7
  80     - Add new patch to fix the 'reverse' bug
  81
  82     Series-version: 8
  83
  84     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  85     Reported-by: York Sun <yorksun@freescale.com>
  86     Signed-off-by: Simon Glass <sjg@chromium.org>
  87
  88 """,
  89 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  90 Author: Simon Glass <sjg@chromium.org>
  91 Date:   Sat Aug 9 11:44:32 2014 -0600
  92
  93     patman: Fix indentation in terminal.py
  94
  95     This code came from a different project with 2-character indentation. Fix
  96     it for U-Boot.
  97
  98     Series-changes: 6
  99     - Add new patch to fix indentation in teminal.py
 100
 101     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
 102     Signed-off-by: Simon Glass <sjg@chromium.org>
 103
 104 """,
 105 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 106 Author: Simon Glass <sjg@chromium.org>
 107 Date:   Sat Aug 9 11:08:24 2014 -0600
 108
 109     patman: Correct unit tests to run correctly
 110
 111     It seems that doctest behaves differently now, and some of the unit tests
 112     do not run. Adjust the tests to work correctly.
 113
 114      ./tools/patman/patman --test
 115     <unittest.result.TestResult run=10 errors=0 failures=0>
 116
 117     Series-changes: 6
 118     - Add new patch to fix patman unit tests
 119
 120     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 121
 122 """,
 123 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 124 Author: Simon Glass <sjg@chromium.org>
 125 Date:   Sat Aug 9 12:06:02 2014 -0600
 126
 127     patman: Remove the -a option
 128
 129     It seems that this is no longer needed, since checkpatch.pl will catch
 130     whitespace problems in patches. Also the option is not widely used, so
 131     it seems safe to just remove it.
 132
 133     Series-changes: 6
 134     - Add new patch to remove patman's -a option
 135
 136     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 137     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 138
 139 """,
 140 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 141 Author: Simon Glass <sjg@chromium.org>
 142 Date:   Thu Aug 14 21:50:52 2014 -0600
 143
 144     patman: Use --no-pager' to stop git from forking a pager
 145
 146 """,
 147 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 148 Author: Simon Glass <sjg@chromium.org>
 149 Date:   Fri Aug 22 15:57:39 2014 -0600
 150
 151     patman: Avoid changing the order of tags
 152
 153     patman collects tags that it sees in the commit and places them nicely
 154     sorted at the end of the patch. However, this is not really necessary and
 155     in fact is apparently not desirable.
 156
 157     Series-changes: 9
 158     - Add new patch to avoid changing the order of tags
 159
 160     Series-version: 9
 161
 162     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 163     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 164 """]
 165
 166 TEST_BRANCH = '__testbranch'
 167
 168 class TestFunctional(unittest.TestCase):
 169     """Functional test for buildman.
 170
 171     This aims to test from just below the invocation of buildman (parsing
 172     of arguments) to 'make' and 'git' invocation. It is not a true
 173     emd-to-end test, as it mocks git, make and the tool chain. But this
 174     makes it easier to detect when the builder is doing the wrong thing,
 175     since in many cases this test code will fail. For example, only a
 176     very limited subset of 'git' arguments is supported - anything
 177     unexpected will fail.
 178     """
 179     def setUp(self):
 180         self._base_dir = tempfile.mkdtemp()
 181         self._git_dir = os.path.join(self._base_dir, 'src')
 182         self._buildman_pathname = sys.argv[0]
 183         self._buildman_dir = os.path.dirname(sys.argv[0])
 184         command.test_result = self._HandleCommand
 185         self.setupToolchains()
 186         self._toolchains.Add('arm-gcc', test=False)
 187         self._toolchains.Add('powerpc-gcc', test=False)
 188         bsettings.Setup(None)
 189         bsettings.AddFile(settings_data)
 190         self._boards = board.Boards()
 191         for brd in boards:
 192             self._boards.AddBoard(board.Board(*brd))
 193
 194         # Directories where the source been cloned
 195         self._clone_dirs = []
 196         self._commits = len(commit_shortlog.splitlines()) + 1
 197         self._total_builds = self._commits * len(boards)
 198
 199         # Number of calls to make
 200         self._make_calls = 0
 201
 202         # Map of [board, commit] to error messages
 203         self._error = {}
 204
 205         self._test_branch = TEST_BRANCH
 206
 207         # Avoid sending any output and clear all terminal output
 208         terminal.SetPrintTestMode()
 209         terminal.GetPrintTestLines()
 210
 211     def tearDown(self):
 212         shutil.rmtree(self._base_dir)
 213
 214     def setupToolchains(self):
 215         self._toolchains = toolchain.Toolchains()
 216         self._toolchains.Add('gcc', test=False)
 217
 218     def _RunBuildman(self, *args):
 219         return command.RunPipe([[self._buildman_pathname] + list(args)],
 220                 capture=True, capture_stderr=True)
 221
 222     def _RunControl(self, *args, **kwargs):
 223         sys.argv = [sys.argv[0]] + list(args)
 224         options, args = cmdline.ParseArgs()
 225         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 226                 make_func=self._HandleMake, boards=self._boards,
 227                 clean_dir=kwargs.get('clean_dir', True))
 228         self._builder = control.builder
 229         return result
 230
 231     def testFullHelp(self):
 232         command.test_result = None
 233         result = self._RunBuildman('-H')
 234         help_file = os.path.join(self._buildman_dir, 'README')
 235         self.assertEqual(len(result.stdout), os.path.getsize(help_file))
 236         self.assertEqual(0, len(result.stderr))
 237         self.assertEqual(0, result.return_code)
 238
 239     def testHelp(self):
 240         command.test_result = None
 241         result = self._RunBuildman('-h')
 242         help_file = os.path.join(self._buildman_dir, 'README')
 243         self.assertTrue(len(result.stdout) > 1000)
 244         self.assertEqual(0, len(result.stderr))
 245         self.assertEqual(0, result.return_code)
 246
 247     def testGitSetup(self):
 248         """Test gitutils.Setup(), from outside the module itself"""
 249         command.test_result = command.CommandResult(return_code=1)
 250         gitutil.Setup()
 251         self.assertEqual(gitutil.use_no_decorate, False)
 252
 253         command.test_result = command.CommandResult(return_code=0)
 254         gitutil.Setup()
 255         self.assertEqual(gitutil.use_no_decorate, True)
 256
 257     def _HandleCommandGitLog(self, args):
 258         if '-n0' in args:
 259             return command.CommandResult(return_code=0)
 260         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 261             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 262         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 263             if args[-1] == self._test_branch:
 264                 count = int(args[3][2:])
 265                 return command.CommandResult(return_code=0,
 266                                             stdout=''.join(commit_log[:count]))
 267
 268         # Not handled, so abort
 269         print 'git log', args
 270         sys.exit(1)
 271
 272     def _HandleCommandGitConfig(self, args):
 273         config = args[0]
 274         if config == 'sendemail.aliasesfile':
 275             return command.CommandResult(return_code=0)
 276         elif config.startswith('branch.badbranch'):
 277             return command.CommandResult(return_code=1)
 278         elif config == 'branch.%s.remote' % self._test_branch:
 279             return command.CommandResult(return_code=0, stdout='upstream\n')
 280         elif config == 'branch.%s.merge' % self._test_branch:
 281             return command.CommandResult(return_code=0,
 282                                          stdout='refs/heads/master\n')
 283
 284         # Not handled, so abort
 285         print 'git config', args
 286         sys.exit(1)
 287
 288     def _HandleCommandGit(self, in_args):
 289         """Handle execution of a git command
 290
 291         This uses a hacked-up parser.
 292
 293         Args:
 294             in_args: Arguments after 'git' from the command line
 295         """
 296         git_args = []           # Top-level arguments to git itself
 297         sub_cmd = None          # Git sub-command selected
 298         args = []               # Arguments to the git sub-command
 299         for arg in in_args:
 300             if sub_cmd:
 301                 args.append(arg)
 302             elif arg[0] == '-':
 303                 git_args.append(arg)
 304             else:
 305                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 306                     git_args.append(arg)
 307                 else:
 308                     sub_cmd = arg
 309         if sub_cmd == 'config':
 310             return self._HandleCommandGitConfig(args)
 311         elif sub_cmd == 'log':
 312             return self._HandleCommandGitLog(args)
 313         elif sub_cmd == 'clone':
 314             return command.CommandResult(return_code=0)
 315         elif sub_cmd == 'checkout':
 316             return command.CommandResult(return_code=0)
 317
 318         # Not handled, so abort
 319         print 'git', git_args, sub_cmd, args
 320         sys.exit(1)
 321
 322     def _HandleCommandNm(self, args):
 323         return command.CommandResult(return_code=0)
 324
 325     def _HandleCommandObjdump(self, args):
 326         return command.CommandResult(return_code=0)
 327
 328     def _HandleCommandSize(self, args):
 329         return command.CommandResult(return_code=0)
 330
 331     def _HandleCommand(self, **kwargs):
 332         """Handle a command execution.
 333
 334         The command is in kwargs['pipe-list'], as a list of pipes, each a
 335         list of commands. The command should be emulated as required for
 336         testing purposes.
 337
 338         Returns:
 339             A CommandResult object
 340         """
 341         pipe_list = kwargs['pipe_list']
 342         wc = False
 343         if len(pipe_list) != 1:
 344             if pipe_list[1] == ['wc', '-l']:
 345                 wc = True
 346             else:
 347                 print 'invalid pipe', kwargs
 348                 sys.exit(1)
 349         cmd = pipe_list[0][0]
 350         args = pipe_list[0][1:]
 351         result = None
 352         if cmd == 'git':
 353             result = self._HandleCommandGit(args)
 354         elif cmd == './scripts/show-gnu-make':
 355             return command.CommandResult(return_code=0, stdout='make')
 356         elif cmd.endswith('nm'):
 357             return self._HandleCommandNm(args)
 358         elif cmd.endswith('objdump'):
 359             return self._HandleCommandObjdump(args)
 360         elif cmd.endswith( 'size'):
 361             return self._HandleCommandSize(args)
 362
 363         if not result:
 364             # Not handled, so abort
 365             print 'unknown command', kwargs
 366             sys.exit(1)
 367
 368         if wc:
 369             result.stdout = len(result.stdout.splitlines())
 370         return result
 371
 372     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 373         """Handle execution of 'make'
 374
 375         Args:
 376             commit: Commit object that is being built
 377             brd: Board object that is being built
 378             stage: Stage that we are at (mrproper, config, build)
 379             cwd: Directory where make should be run
 380             args: Arguments to pass to make
 381             kwargs: Arguments to pass to command.RunPipe()
 382         """
 383         self._make_calls += 1
 384         if stage == 'mrproper':
 385             return command.CommandResult(return_code=0)
 386         elif stage == 'config':
 387             return command.CommandResult(return_code=0,
 388                     combined='Test configuration complete')
 389         elif stage == 'build':
 390             stderr = ''
 391             if type(commit) is not str:
 392                 stderr = self._error.get((brd.target, commit.sequence))
 393             if stderr:
 394                 return command.CommandResult(return_code=1, stderr=stderr)
 395             return command.CommandResult(return_code=0)
 396
 397         # Not handled, so abort
 398         print 'make', stage
 399         sys.exit(1)
 400
 401     # Example function to print output lines
 402     def print_lines(self, lines):
 403         print len(lines)
 404         for line in lines:
 405             print line
 406         #self.print_lines(terminal.GetPrintTestLines())
 407
 408     def testNoBoards(self):
 409         """Test that buildman aborts when there are no boards"""
 410         self._boards = board.Boards()
 411         with self.assertRaises(SystemExit):
 412             self._RunControl()
 413
 414     def testCurrentSource(self):
 415         """Very simple test to invoke buildman on the current source"""
 416         self.setupToolchains();
 417         self._RunControl()
 418         lines = terminal.GetPrintTestLines()
 419         self.assertIn('Building current source for %d boards' % len(boards),
 420                       lines[0].text)
 421
 422     def testBadBranch(self):
 423         """Test that we can detect an invalid branch"""
 424         with self.assertRaises(ValueError):
 425             self._RunControl('-b', 'badbranch')
 426
 427     def testBadToolchain(self):
 428         """Test that missing toolchains are detected"""
 429         self.setupToolchains();
 430         ret_code = self._RunControl('-b', TEST_BRANCH)
 431         lines = terminal.GetPrintTestLines()
 432
 433         # Buildman always builds the upstream commit as well
 434         self.assertIn('Building %d commits for %d boards' %
 435                 (self._commits, len(boards)), lines[0].text)
 436         self.assertEqual(self._builder.count, self._total_builds)
 437
 438         # Only sandbox should succeed, the others don't have toolchains
 439         self.assertEqual(self._builder.fail,
 440                          self._total_builds - self._commits)
 441         self.assertEqual(ret_code, 128)
 442
 443         for commit in range(self._commits):
 444             for board in self._boards.GetList():
 445                 if board.arch != 'sandbox':
 446                   errfile = self._builder.GetErrFile(commit, board.target)
 447                   fd = open(errfile)
 448                   self.assertEqual(fd.readlines(),
 449                           ['No tool chain for %s\n' % board.arch])
 450                   fd.close()
 451
 452     def testBranch(self):
 453         """Test building a branch with all toolchains present"""
 454         self._RunControl('-b', TEST_BRANCH)
 455         self.assertEqual(self._builder.count, self._total_builds)
 456         self.assertEqual(self._builder.fail, 0)
 457
 458     def testCount(self):
 459         """Test building a specific number of commitst"""
 460         self._RunControl('-b', TEST_BRANCH, '-c2')
 461         self.assertEqual(self._builder.count, 2 * len(boards))
 462         self.assertEqual(self._builder.fail, 0)
 463         # Each board has a mrproper, config, and then one make per commit
 464         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 465
 466     def testIncremental(self):
 467         """Test building a branch twice - the second time should do nothing"""
 468         self._RunControl('-b', TEST_BRANCH)
 469
 470         # Each board has a mrproper, config, and then one make per commit
 471         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 472         self._make_calls = 0
 473         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 474         self.assertEqual(self._make_calls, 0)
 475         self.assertEqual(self._builder.count, self._total_builds)
 476         self.assertEqual(self._builder.fail, 0)
 477
 478     def testForceBuild(self):
 479         """The -f flag should force a rebuild"""
 480         self._RunControl('-b', TEST_BRANCH)
 481         self._make_calls = 0
 482         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 483         # Each board has a mrproper, config, and then one make per commit
 484         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 485
 486     def testForceReconfigure(self):
 487         """The -f flag should force a rebuild"""
 488         self._RunControl('-b', TEST_BRANCH, '-C')
 489         # Each commit has a mrproper, config and make
 490         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 491
 492     def testErrors(self):
 493         """Test handling of build errors"""
 494         self._error['board2', 1] = 'fred\n'
 495         self._RunControl('-b', TEST_BRANCH)
 496         self.assertEqual(self._builder.count, self._total_builds)
 497         self.assertEqual(self._builder.fail, 1)
 498
 499         # Remove the error. This should have no effect since the commit will
 500         # not be rebuilt
 501         del self._error['board2', 1]
 502         self._make_calls = 0
 503         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 504         self.assertEqual(self._builder.count, self._total_builds)
 505         self.assertEqual(self._make_calls, 0)
 506         self.assertEqual(self._builder.fail, 1)
 507
 508         # Now use the -F flag to force rebuild of the bad commit
 509         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 510         self.assertEqual(self._builder.count, self._total_builds)
 511         self.assertEqual(self._builder.fail, 0)
 512         self.assertEqual(self._make_calls, 3)
 513
 514     def testBranchWithSlash(self):
 515         """Test building a branch with a '/' in the name"""
 516         self._test_branch = '/__dev/__testbranch'
 517         self._RunControl('-b', self._test_branch, clean_dir=False)
 518         self.assertEqual(self._builder.count, self._total_builds)
 519         self.assertEqual(self._builder.fail, 0)