tools/buildman/func_test.py

   1 #
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4 # SPDX-License-Identifier:      GPL-2.0+
   5 #
   6
   7 import os
   8 import shutil
   9 import sys
  10 import tempfile
  11 import unittest
  12
  13 import board
  14 import bsettings
  15 import cmdline
  16 import command
  17 import control
  18 import gitutil
  19 import terminal
  20 import toolchain
  21
  22 settings_data = '''
  23 # Buildman settings file
  24
  25 [toolchain]
  26
  27 [toolchain-alias]
  28
  29 [make-flags]
  30 src=/home/sjg/c/src
  31 chroot=/home/sjg/c/chroot
  32 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  33 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  34 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  35 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  36 '''
  37
  38 boards = [
  39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  40     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  41     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  42     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  43 ]
  44
  45 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  46 39403bb patman: Use --no-pager' to stop git from forking a pager
  47 db6e6f2 patman: Remove the -a option
  48 f2ccf03 patman: Correct unit tests to run correctly
  49 1d097f9 patman: Fix indentation in terminal.py
  50 d073747 patman: Support the 'reverse' option for 'git log
  51 """
  52
  53 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  54 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  55 Date:   Fri Aug 22 19:12:41 2014 +0900
  56
  57     buildman: refactor help message
  58
  59     "buildman [options]" is displayed by default.
  60
  61     Append the rest of help messages to parser.usage
  62     instead of replacing it.
  63
  64     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  65     Drop it from the usage.
  66
  67     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  68 """,
  69 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  70 Author: Simon Glass <sjg@chromium.org>
  71 Date:   Thu Aug 14 16:48:25 2014 -0600
  72
  73     patman: Support the 'reverse' option for 'git log'
  74
  75     This option is currently not supported, but needs to be, for buildman to
  76     operate as expected.
  77
  78     Series-changes: 7
  79     - Add new patch to fix the 'reverse' bug
  80
  81     Series-version: 8
  82
  83     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  84     Reported-by: York Sun <yorksun@freescale.com>
  85     Signed-off-by: Simon Glass <sjg@chromium.org>
  86
  87 """,
  88 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  89 Author: Simon Glass <sjg@chromium.org>
  90 Date:   Sat Aug 9 11:44:32 2014 -0600
  91
  92     patman: Fix indentation in terminal.py
  93
  94     This code came from a different project with 2-character indentation. Fix
  95     it for U-Boot.
  96
  97     Series-changes: 6
  98     - Add new patch to fix indentation in teminal.py
  99
 100     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
 101     Signed-off-by: Simon Glass <sjg@chromium.org>
 102
 103 """,
 104 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 105 Author: Simon Glass <sjg@chromium.org>
 106 Date:   Sat Aug 9 11:08:24 2014 -0600
 107
 108     patman: Correct unit tests to run correctly
 109
 110     It seems that doctest behaves differently now, and some of the unit tests
 111     do not run. Adjust the tests to work correctly.
 112
 113      ./tools/patman/patman --test
 114     <unittest.result.TestResult run=10 errors=0 failures=0>
 115
 116     Series-changes: 6
 117     - Add new patch to fix patman unit tests
 118
 119     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 120
 121 """,
 122 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 123 Author: Simon Glass <sjg@chromium.org>
 124 Date:   Sat Aug 9 12:06:02 2014 -0600
 125
 126     patman: Remove the -a option
 127
 128     It seems that this is no longer needed, since checkpatch.pl will catch
 129     whitespace problems in patches. Also the option is not widely used, so
 130     it seems safe to just remove it.
 131
 132     Series-changes: 6
 133     - Add new patch to remove patman's -a option
 134
 135     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 136     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 137
 138 """,
 139 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 140 Author: Simon Glass <sjg@chromium.org>
 141 Date:   Thu Aug 14 21:50:52 2014 -0600
 142
 143     patman: Use --no-pager' to stop git from forking a pager
 144
 145 """,
 146 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 147 Author: Simon Glass <sjg@chromium.org>
 148 Date:   Fri Aug 22 15:57:39 2014 -0600
 149
 150     patman: Avoid changing the order of tags
 151
 152     patman collects tags that it sees in the commit and places them nicely
 153     sorted at the end of the patch. However, this is not really necessary and
 154     in fact is apparently not desirable.
 155
 156     Series-changes: 9
 157     - Add new patch to avoid changing the order of tags
 158
 159     Series-version: 9
 160
 161     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 162     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 163 """]
 164
 165 TEST_BRANCH = '__testbranch'
 166
 167 class TestFunctional(unittest.TestCase):
 168     """Functional test for buildman.
 169
 170     This aims to test from just below the invocation of buildman (parsing
 171     of arguments) to 'make' and 'git' invocation. It is not a true
 172     emd-to-end test, as it mocks git, make and the tool chain. But this
 173     makes it easier to detect when the builder is doing the wrong thing,
 174     since in many cases this test code will fail. For example, only a
 175     very limited subset of 'git' arguments is supported - anything
 176     unexpected will fail.
 177     """
 178     def setUp(self):
 179         self._base_dir = tempfile.mkdtemp()
 180         self._git_dir = os.path.join(self._base_dir, 'src')
 181         self._buildman_pathname = sys.argv[0]
 182         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 183         command.test_result = self._HandleCommand
 184         self.setupToolchains()
 185         self._toolchains.Add('arm-gcc', test=False)
 186         self._toolchains.Add('powerpc-gcc', test=False)
 187         bsettings.Setup(None)
 188         bsettings.AddFile(settings_data)
 189         self._boards = board.Boards()
 190         for brd in boards:
 191             self._boards.AddBoard(board.Board(*brd))
 192
 193         # Directories where the source been cloned
 194         self._clone_dirs = []
 195         self._commits = len(commit_shortlog.splitlines()) + 1
 196         self._total_builds = self._commits * len(boards)
 197
 198         # Number of calls to make
 199         self._make_calls = 0
 200
 201         # Map of [board, commit] to error messages
 202         self._error = {}
 203
 204         self._test_branch = TEST_BRANCH
 205
 206         # Avoid sending any output and clear all terminal output
 207         terminal.SetPrintTestMode()
 208         terminal.GetPrintTestLines()
 209
 210     def tearDown(self):
 211         shutil.rmtree(self._base_dir)
 212
 213     def setupToolchains(self):
 214         self._toolchains = toolchain.Toolchains()
 215         self._toolchains.Add('gcc', test=False)
 216
 217     def _RunBuildman(self, *args):
 218         return command.RunPipe([[self._buildman_pathname] + list(args)],
 219                 capture=True, capture_stderr=True)
 220
 221     def _RunControl(self, *args, **kwargs):
 222         sys.argv = [sys.argv[0]] + list(args)
 223         options, args = cmdline.ParseArgs()
 224         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 225                 make_func=self._HandleMake, boards=self._boards,
 226                 clean_dir=kwargs.get('clean_dir', True))
 227         self._builder = control.builder
 228         return result
 229
 230     def testFullHelp(self):
 231         command.test_result = None
 232         result = self._RunBuildman('-H')
 233         help_file = os.path.join(self._buildman_dir, 'README')
 234         self.assertEqual(len(result.stdout), os.path.getsize(help_file))
 235         self.assertEqual(0, len(result.stderr))
 236         self.assertEqual(0, result.return_code)
 237
 238     def testHelp(self):
 239         command.test_result = None
 240         result = self._RunBuildman('-h')
 241         help_file = os.path.join(self._buildman_dir, 'README')
 242         self.assertTrue(len(result.stdout) > 1000)
 243         self.assertEqual(0, len(result.stderr))
 244         self.assertEqual(0, result.return_code)
 245
 246     def testGitSetup(self):
 247         """Test gitutils.Setup(), from outside the module itself"""
 248         command.test_result = command.CommandResult(return_code=1)
 249         gitutil.Setup()
 250         self.assertEqual(gitutil.use_no_decorate, False)
 251
 252         command.test_result = command.CommandResult(return_code=0)
 253         gitutil.Setup()
 254         self.assertEqual(gitutil.use_no_decorate, True)
 255
 256     def _HandleCommandGitLog(self, args):
 257         if args[-1] == '--':
 258             args = args[:-1]
 259         if '-n0' in args:
 260             return command.CommandResult(return_code=0)
 261         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 262             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 263         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 264             if args[-1] == self._test_branch:
 265                 count = int(args[3][2:])
 266                 return command.CommandResult(return_code=0,
 267                                             stdout=''.join(commit_log[:count]))
 268
 269         # Not handled, so abort
 270         print 'git log', args
 271         sys.exit(1)
 272
 273     def _HandleCommandGitConfig(self, args):
 274         config = args[0]
 275         if config == 'sendemail.aliasesfile':
 276             return command.CommandResult(return_code=0)
 277         elif config.startswith('branch.badbranch'):
 278             return command.CommandResult(return_code=1)
 279         elif config == 'branch.%s.remote' % self._test_branch:
 280             return command.CommandResult(return_code=0, stdout='upstream\n')
 281         elif config == 'branch.%s.merge' % self._test_branch:
 282             return command.CommandResult(return_code=0,
 283                                          stdout='refs/heads/master\n')
 284
 285         # Not handled, so abort
 286         print 'git config', args
 287         sys.exit(1)
 288
 289     def _HandleCommandGit(self, in_args):
 290         """Handle execution of a git command
 291
 292         This uses a hacked-up parser.
 293
 294         Args:
 295             in_args: Arguments after 'git' from the command line
 296         """
 297         git_args = []           # Top-level arguments to git itself
 298         sub_cmd = None          # Git sub-command selected
 299         args = []               # Arguments to the git sub-command
 300         for arg in in_args:
 301             if sub_cmd:
 302                 args.append(arg)
 303             elif arg[0] == '-':
 304                 git_args.append(arg)
 305             else:
 306                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 307                     git_args.append(arg)
 308                 else:
 309                     sub_cmd = arg
 310         if sub_cmd == 'config':
 311             return self._HandleCommandGitConfig(args)
 312         elif sub_cmd == 'log':
 313             return self._HandleCommandGitLog(args)
 314         elif sub_cmd == 'clone':
 315             return command.CommandResult(return_code=0)
 316         elif sub_cmd == 'checkout':
 317             return command.CommandResult(return_code=0)
 318
 319         # Not handled, so abort
 320         print 'git', git_args, sub_cmd, args
 321         sys.exit(1)
 322
 323     def _HandleCommandNm(self, args):
 324         return command.CommandResult(return_code=0)
 325
 326     def _HandleCommandObjdump(self, args):
 327         return command.CommandResult(return_code=0)
 328
 329     def _HandleCommandSize(self, args):
 330         return command.CommandResult(return_code=0)
 331
 332     def _HandleCommand(self, **kwargs):
 333         """Handle a command execution.
 334
 335         The command is in kwargs['pipe-list'], as a list of pipes, each a
 336         list of commands. The command should be emulated as required for
 337         testing purposes.
 338
 339         Returns:
 340             A CommandResult object
 341         """
 342         pipe_list = kwargs['pipe_list']
 343         wc = False
 344         if len(pipe_list) != 1:
 345             if pipe_list[1] == ['wc', '-l']:
 346                 wc = True
 347             else:
 348                 print 'invalid pipe', kwargs
 349                 sys.exit(1)
 350         cmd = pipe_list[0][0]
 351         args = pipe_list[0][1:]
 352         result = None
 353         if cmd == 'git':
 354             result = self._HandleCommandGit(args)
 355         elif cmd == './scripts/show-gnu-make':
 356             return command.CommandResult(return_code=0, stdout='make')
 357         elif cmd.endswith('nm'):
 358             return self._HandleCommandNm(args)
 359         elif cmd.endswith('objdump'):
 360             return self._HandleCommandObjdump(args)
 361         elif cmd.endswith( 'size'):
 362             return self._HandleCommandSize(args)
 363
 364         if not result:
 365             # Not handled, so abort
 366             print 'unknown command', kwargs
 367             sys.exit(1)
 368
 369         if wc:
 370             result.stdout = len(result.stdout.splitlines())
 371         return result
 372
 373     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 374         """Handle execution of 'make'
 375
 376         Args:
 377             commit: Commit object that is being built
 378             brd: Board object that is being built
 379             stage: Stage that we are at (mrproper, config, build)
 380             cwd: Directory where make should be run
 381             args: Arguments to pass to make
 382             kwargs: Arguments to pass to command.RunPipe()
 383         """
 384         self._make_calls += 1
 385         if stage == 'mrproper':
 386             return command.CommandResult(return_code=0)
 387         elif stage == 'config':
 388             return command.CommandResult(return_code=0,
 389                     combined='Test configuration complete')
 390         elif stage == 'build':
 391             stderr = ''
 392             if type(commit) is not str:
 393                 stderr = self._error.get((brd.target, commit.sequence))
 394             if stderr:
 395                 return command.CommandResult(return_code=1, stderr=stderr)
 396             return command.CommandResult(return_code=0)
 397
 398         # Not handled, so abort
 399         print 'make', stage
 400         sys.exit(1)
 401
 402     # Example function to print output lines
 403     def print_lines(self, lines):
 404         print len(lines)
 405         for line in lines:
 406             print line
 407         #self.print_lines(terminal.GetPrintTestLines())
 408
 409     def testNoBoards(self):
 410         """Test that buildman aborts when there are no boards"""
 411         self._boards = board.Boards()
 412         with self.assertRaises(SystemExit):
 413             self._RunControl()
 414
 415     def testCurrentSource(self):
 416         """Very simple test to invoke buildman on the current source"""
 417         self.setupToolchains();
 418         self._RunControl()
 419         lines = terminal.GetPrintTestLines()
 420         self.assertIn('Building current source for %d boards' % len(boards),
 421                       lines[0].text)
 422
 423     def testBadBranch(self):
 424         """Test that we can detect an invalid branch"""
 425         with self.assertRaises(ValueError):
 426             self._RunControl('-b', 'badbranch')
 427
 428     def testBadToolchain(self):
 429         """Test that missing toolchains are detected"""
 430         self.setupToolchains();
 431         ret_code = self._RunControl('-b', TEST_BRANCH)
 432         lines = terminal.GetPrintTestLines()
 433
 434         # Buildman always builds the upstream commit as well
 435         self.assertIn('Building %d commits for %d boards' %
 436                 (self._commits, len(boards)), lines[0].text)
 437         self.assertEqual(self._builder.count, self._total_builds)
 438
 439         # Only sandbox should succeed, the others don't have toolchains
 440         self.assertEqual(self._builder.fail,
 441                          self._total_builds - self._commits)
 442         self.assertEqual(ret_code, 128)
 443
 444         for commit in range(self._commits):
 445             for board in self._boards.GetList():
 446                 if board.arch != 'sandbox':
 447                   errfile = self._builder.GetErrFile(commit, board.target)
 448                   fd = open(errfile)
 449                   self.assertEqual(fd.readlines(),
 450                           ['No tool chain for %s\n' % board.arch])
 451                   fd.close()
 452
 453     def testBranch(self):
 454         """Test building a branch with all toolchains present"""
 455         self._RunControl('-b', TEST_BRANCH)
 456         self.assertEqual(self._builder.count, self._total_builds)
 457         self.assertEqual(self._builder.fail, 0)
 458
 459     def testCount(self):
 460         """Test building a specific number of commitst"""
 461         self._RunControl('-b', TEST_BRANCH, '-c2')
 462         self.assertEqual(self._builder.count, 2 * len(boards))
 463         self.assertEqual(self._builder.fail, 0)
 464         # Each board has a mrproper, config, and then one make per commit
 465         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 466
 467     def testIncremental(self):
 468         """Test building a branch twice - the second time should do nothing"""
 469         self._RunControl('-b', TEST_BRANCH)
 470
 471         # Each board has a mrproper, config, and then one make per commit
 472         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 473         self._make_calls = 0
 474         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 475         self.assertEqual(self._make_calls, 0)
 476         self.assertEqual(self._builder.count, self._total_builds)
 477         self.assertEqual(self._builder.fail, 0)
 478
 479     def testForceBuild(self):
 480         """The -f flag should force a rebuild"""
 481         self._RunControl('-b', TEST_BRANCH)
 482         self._make_calls = 0
 483         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 484         # Each board has a mrproper, config, and then one make per commit
 485         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 486
 487     def testForceReconfigure(self):
 488         """The -f flag should force a rebuild"""
 489         self._RunControl('-b', TEST_BRANCH, '-C')
 490         # Each commit has a mrproper, config and make
 491         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 492
 493     def testErrors(self):
 494         """Test handling of build errors"""
 495         self._error['board2', 1] = 'fred\n'
 496         self._RunControl('-b', TEST_BRANCH)
 497         self.assertEqual(self._builder.count, self._total_builds)
 498         self.assertEqual(self._builder.fail, 1)
 499
 500         # Remove the error. This should have no effect since the commit will
 501         # not be rebuilt
 502         del self._error['board2', 1]
 503         self._make_calls = 0
 504         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 505         self.assertEqual(self._builder.count, self._total_builds)
 506         self.assertEqual(self._make_calls, 0)
 507         self.assertEqual(self._builder.fail, 1)
 508
 509         # Now use the -F flag to force rebuild of the bad commit
 510         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 511         self.assertEqual(self._builder.count, self._total_builds)
 512         self.assertEqual(self._builder.fail, 0)
 513         self.assertEqual(self._make_calls, 3)
 514
 515     def testBranchWithSlash(self):
 516         """Test building a branch with a '/' in the name"""
 517         self._test_branch = '/__dev/__testbranch'
 518         self._RunControl('-b', self._test_branch, clean_dir=False)
 519         self.assertEqual(self._builder.count, self._total_builds)
 520         self.assertEqual(self._builder.fail, 0)