tools/buildman/func_test.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4
   5 import os
   6 import shutil
   7 import sys
   8 import tempfile
   9 import unittest
  10
  11 import board
  12 import bsettings
  13 import cmdline
  14 import command
  15 import control
  16 import gitutil
  17 import terminal
  18 import toolchain
  19
  20 settings_data = '''
  21 # Buildman settings file
  22
  23 [toolchain]
  24
  25 [toolchain-alias]
  26
  27 [make-flags]
  28 src=/home/sjg/c/src
  29 chroot=/home/sjg/c/chroot
  30 vboot=VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  31 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  32 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  33 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  34 '''
  35
  36 boards = [
  37     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  38     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  39     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  40     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  41 ]
  42
  43 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  44 39403bb patman: Use --no-pager' to stop git from forking a pager
  45 db6e6f2 patman: Remove the -a option
  46 f2ccf03 patman: Correct unit tests to run correctly
  47 1d097f9 patman: Fix indentation in terminal.py
  48 d073747 patman: Support the 'reverse' option for 'git log
  49 """
  50
  51 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  52 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  53 Date:   Fri Aug 22 19:12:41 2014 +0900
  54
  55     buildman: refactor help message
  56
  57     "buildman [options]" is displayed by default.
  58
  59     Append the rest of help messages to parser.usage
  60     instead of replacing it.
  61
  62     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  63     Drop it from the usage.
  64
  65     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  66 """,
  67 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  68 Author: Simon Glass <sjg@chromium.org>
  69 Date:   Thu Aug 14 16:48:25 2014 -0600
  70
  71     patman: Support the 'reverse' option for 'git log'
  72
  73     This option is currently not supported, but needs to be, for buildman to
  74     operate as expected.
  75
  76     Series-changes: 7
  77     - Add new patch to fix the 'reverse' bug
  78
  79     Series-version: 8
  80
  81     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  82     Reported-by: York Sun <yorksun@freescale.com>
  83     Signed-off-by: Simon Glass <sjg@chromium.org>
  84
  85 """,
  86 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  87 Author: Simon Glass <sjg@chromium.org>
  88 Date:   Sat Aug 9 11:44:32 2014 -0600
  89
  90     patman: Fix indentation in terminal.py
  91
  92     This code came from a different project with 2-character indentation. Fix
  93     it for U-Boot.
  94
  95     Series-changes: 6
  96     - Add new patch to fix indentation in teminal.py
  97
  98     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
  99     Signed-off-by: Simon Glass <sjg@chromium.org>
 100
 101 """,
 102 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 103 Author: Simon Glass <sjg@chromium.org>
 104 Date:   Sat Aug 9 11:08:24 2014 -0600
 105
 106     patman: Correct unit tests to run correctly
 107
 108     It seems that doctest behaves differently now, and some of the unit tests
 109     do not run. Adjust the tests to work correctly.
 110
 111      ./tools/patman/patman --test
 112     <unittest.result.TestResult run=10 errors=0 failures=0>
 113
 114     Series-changes: 6
 115     - Add new patch to fix patman unit tests
 116
 117     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 118
 119 """,
 120 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 121 Author: Simon Glass <sjg@chromium.org>
 122 Date:   Sat Aug 9 12:06:02 2014 -0600
 123
 124     patman: Remove the -a option
 125
 126     It seems that this is no longer needed, since checkpatch.pl will catch
 127     whitespace problems in patches. Also the option is not widely used, so
 128     it seems safe to just remove it.
 129
 130     Series-changes: 6
 131     - Add new patch to remove patman's -a option
 132
 133     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 134     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 135
 136 """,
 137 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 138 Author: Simon Glass <sjg@chromium.org>
 139 Date:   Thu Aug 14 21:50:52 2014 -0600
 140
 141     patman: Use --no-pager' to stop git from forking a pager
 142
 143 """,
 144 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 145 Author: Simon Glass <sjg@chromium.org>
 146 Date:   Fri Aug 22 15:57:39 2014 -0600
 147
 148     patman: Avoid changing the order of tags
 149
 150     patman collects tags that it sees in the commit and places them nicely
 151     sorted at the end of the patch. However, this is not really necessary and
 152     in fact is apparently not desirable.
 153
 154     Series-changes: 9
 155     - Add new patch to avoid changing the order of tags
 156
 157     Series-version: 9
 158
 159     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 160     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 161 """]
 162
 163 TEST_BRANCH = '__testbranch'
 164
 165 class TestFunctional(unittest.TestCase):
 166     """Functional test for buildman.
 167
 168     This aims to test from just below the invocation of buildman (parsing
 169     of arguments) to 'make' and 'git' invocation. It is not a true
 170     emd-to-end test, as it mocks git, make and the tool chain. But this
 171     makes it easier to detect when the builder is doing the wrong thing,
 172     since in many cases this test code will fail. For example, only a
 173     very limited subset of 'git' arguments is supported - anything
 174     unexpected will fail.
 175     """
 176     def setUp(self):
 177         self._base_dir = tempfile.mkdtemp()
 178         self._git_dir = os.path.join(self._base_dir, 'src')
 179         self._buildman_pathname = sys.argv[0]
 180         self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 181         command.test_result = self._HandleCommand
 182         self.setupToolchains()
 183         self._toolchains.Add('arm-gcc', test=False)
 184         self._toolchains.Add('powerpc-gcc', test=False)
 185         bsettings.Setup(None)
 186         bsettings.AddFile(settings_data)
 187         self._boards = board.Boards()
 188         for brd in boards:
 189             self._boards.AddBoard(board.Board(*brd))
 190
 191         # Directories where the source been cloned
 192         self._clone_dirs = []
 193         self._commits = len(commit_shortlog.splitlines()) + 1
 194         self._total_builds = self._commits * len(boards)
 195
 196         # Number of calls to make
 197         self._make_calls = 0
 198
 199         # Map of [board, commit] to error messages
 200         self._error = {}
 201
 202         self._test_branch = TEST_BRANCH
 203
 204         # Avoid sending any output and clear all terminal output
 205         terminal.SetPrintTestMode()
 206         terminal.GetPrintTestLines()
 207
 208     def tearDown(self):
 209         shutil.rmtree(self._base_dir)
 210
 211     def setupToolchains(self):
 212         self._toolchains = toolchain.Toolchains()
 213         self._toolchains.Add('gcc', test=False)
 214
 215     def _RunBuildman(self, *args):
 216         return command.RunPipe([[self._buildman_pathname] + list(args)],
 217                 capture=True, capture_stderr=True)
 218
 219     def _RunControl(self, *args, **kwargs):
 220         sys.argv = [sys.argv[0]] + list(args)
 221         options, args = cmdline.ParseArgs()
 222         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 223                 make_func=self._HandleMake, boards=self._boards,
 224                 clean_dir=kwargs.get('clean_dir', True))
 225         self._builder = control.builder
 226         return result
 227
 228     def testFullHelp(self):
 229         command.test_result = None
 230         result = self._RunBuildman('-H')
 231         help_file = os.path.join(self._buildman_dir, 'README')
 232         # Remove possible extraneous strings
 233         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 234         gothelp = result.stdout.replace(extra, '')
 235         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 236         self.assertEqual(0, len(result.stderr))
 237         self.assertEqual(0, result.return_code)
 238
 239     def testHelp(self):
 240         command.test_result = None
 241         result = self._RunBuildman('-h')
 242         help_file = os.path.join(self._buildman_dir, 'README')
 243         self.assertTrue(len(result.stdout) > 1000)
 244         self.assertEqual(0, len(result.stderr))
 245         self.assertEqual(0, result.return_code)
 246
 247     def testGitSetup(self):
 248         """Test gitutils.Setup(), from outside the module itself"""
 249         command.test_result = command.CommandResult(return_code=1)
 250         gitutil.Setup()
 251         self.assertEqual(gitutil.use_no_decorate, False)
 252
 253         command.test_result = command.CommandResult(return_code=0)
 254         gitutil.Setup()
 255         self.assertEqual(gitutil.use_no_decorate, True)
 256
 257     def _HandleCommandGitLog(self, args):
 258         if args[-1] == '--':
 259             args = args[:-1]
 260         if '-n0' in args:
 261             return command.CommandResult(return_code=0)
 262         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 263             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 264         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 265             if args[-1] == self._test_branch:
 266                 count = int(args[3][2:])
 267                 return command.CommandResult(return_code=0,
 268                                             stdout=''.join(commit_log[:count]))
 269
 270         # Not handled, so abort
 271         print 'git log', args
 272         sys.exit(1)
 273
 274     def _HandleCommandGitConfig(self, args):
 275         config = args[0]
 276         if config == 'sendemail.aliasesfile':
 277             return command.CommandResult(return_code=0)
 278         elif config.startswith('branch.badbranch'):
 279             return command.CommandResult(return_code=1)
 280         elif config == 'branch.%s.remote' % self._test_branch:
 281             return command.CommandResult(return_code=0, stdout='upstream\n')
 282         elif config == 'branch.%s.merge' % self._test_branch:
 283             return command.CommandResult(return_code=0,
 284                                          stdout='refs/heads/master\n')
 285
 286         # Not handled, so abort
 287         print 'git config', args
 288         sys.exit(1)
 289
 290     def _HandleCommandGit(self, in_args):
 291         """Handle execution of a git command
 292
 293         This uses a hacked-up parser.
 294
 295         Args:
 296             in_args: Arguments after 'git' from the command line
 297         """
 298         git_args = []           # Top-level arguments to git itself
 299         sub_cmd = None          # Git sub-command selected
 300         args = []               # Arguments to the git sub-command
 301         for arg in in_args:
 302             if sub_cmd:
 303                 args.append(arg)
 304             elif arg[0] == '-':
 305                 git_args.append(arg)
 306             else:
 307                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 308                     git_args.append(arg)
 309                 else:
 310                     sub_cmd = arg
 311         if sub_cmd == 'config':
 312             return self._HandleCommandGitConfig(args)
 313         elif sub_cmd == 'log':
 314             return self._HandleCommandGitLog(args)
 315         elif sub_cmd == 'clone':
 316             return command.CommandResult(return_code=0)
 317         elif sub_cmd == 'checkout':
 318             return command.CommandResult(return_code=0)
 319
 320         # Not handled, so abort
 321         print 'git', git_args, sub_cmd, args
 322         sys.exit(1)
 323
 324     def _HandleCommandNm(self, args):
 325         return command.CommandResult(return_code=0)
 326
 327     def _HandleCommandObjdump(self, args):
 328         return command.CommandResult(return_code=0)
 329
 330     def _HandleCommandObjcopy(self, args):
 331         return command.CommandResult(return_code=0)
 332
 333     def _HandleCommandSize(self, args):
 334         return command.CommandResult(return_code=0)
 335
 336     def _HandleCommand(self, **kwargs):
 337         """Handle a command execution.
 338
 339         The command is in kwargs['pipe-list'], as a list of pipes, each a
 340         list of commands. The command should be emulated as required for
 341         testing purposes.
 342
 343         Returns:
 344             A CommandResult object
 345         """
 346         pipe_list = kwargs['pipe_list']
 347         wc = False
 348         if len(pipe_list) != 1:
 349             if pipe_list[1] == ['wc', '-l']:
 350                 wc = True
 351             else:
 352                 print 'invalid pipe', kwargs
 353                 sys.exit(1)
 354         cmd = pipe_list[0][0]
 355         args = pipe_list[0][1:]
 356         result = None
 357         if cmd == 'git':
 358             result = self._HandleCommandGit(args)
 359         elif cmd == './scripts/show-gnu-make':
 360             return command.CommandResult(return_code=0, stdout='make')
 361         elif cmd.endswith('nm'):
 362             return self._HandleCommandNm(args)
 363         elif cmd.endswith('objdump'):
 364             return self._HandleCommandObjdump(args)
 365         elif cmd.endswith('objcopy'):
 366             return self._HandleCommandObjcopy(args)
 367         elif cmd.endswith( 'size'):
 368             return self._HandleCommandSize(args)
 369
 370         if not result:
 371             # Not handled, so abort
 372             print 'unknown command', kwargs
 373             sys.exit(1)
 374
 375         if wc:
 376             result.stdout = len(result.stdout.splitlines())
 377         return result
 378
 379     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 380         """Handle execution of 'make'
 381
 382         Args:
 383             commit: Commit object that is being built
 384             brd: Board object that is being built
 385             stage: Stage that we are at (mrproper, config, build)
 386             cwd: Directory where make should be run
 387             args: Arguments to pass to make
 388             kwargs: Arguments to pass to command.RunPipe()
 389         """
 390         self._make_calls += 1
 391         if stage == 'mrproper':
 392             return command.CommandResult(return_code=0)
 393         elif stage == 'config':
 394             return command.CommandResult(return_code=0,
 395                     combined='Test configuration complete')
 396         elif stage == 'build':
 397             stderr = ''
 398             if type(commit) is not str:
 399                 stderr = self._error.get((brd.target, commit.sequence))
 400             if stderr:
 401                 return command.CommandResult(return_code=1, stderr=stderr)
 402             return command.CommandResult(return_code=0)
 403
 404         # Not handled, so abort
 405         print 'make', stage
 406         sys.exit(1)
 407
 408     # Example function to print output lines
 409     def print_lines(self, lines):
 410         print len(lines)
 411         for line in lines:
 412             print line
 413         #self.print_lines(terminal.GetPrintTestLines())
 414
 415     def testNoBoards(self):
 416         """Test that buildman aborts when there are no boards"""
 417         self._boards = board.Boards()
 418         with self.assertRaises(SystemExit):
 419             self._RunControl()
 420
 421     def testCurrentSource(self):
 422         """Very simple test to invoke buildman on the current source"""
 423         self.setupToolchains();
 424         self._RunControl()
 425         lines = terminal.GetPrintTestLines()
 426         self.assertIn('Building current source for %d boards' % len(boards),
 427                       lines[0].text)
 428
 429     def testBadBranch(self):
 430         """Test that we can detect an invalid branch"""
 431         with self.assertRaises(ValueError):
 432             self._RunControl('-b', 'badbranch')
 433
 434     def testBadToolchain(self):
 435         """Test that missing toolchains are detected"""
 436         self.setupToolchains();
 437         ret_code = self._RunControl('-b', TEST_BRANCH)
 438         lines = terminal.GetPrintTestLines()
 439
 440         # Buildman always builds the upstream commit as well
 441         self.assertIn('Building %d commits for %d boards' %
 442                 (self._commits, len(boards)), lines[0].text)
 443         self.assertEqual(self._builder.count, self._total_builds)
 444
 445         # Only sandbox should succeed, the others don't have toolchains
 446         self.assertEqual(self._builder.fail,
 447                          self._total_builds - self._commits)
 448         self.assertEqual(ret_code, 128)
 449
 450         for commit in range(self._commits):
 451             for board in self._boards.GetList():
 452                 if board.arch != 'sandbox':
 453                   errfile = self._builder.GetErrFile(commit, board.target)
 454                   fd = open(errfile)
 455                   self.assertEqual(fd.readlines(),
 456                           ['No tool chain for %s\n' % board.arch])
 457                   fd.close()
 458
 459     def testBranch(self):
 460         """Test building a branch with all toolchains present"""
 461         self._RunControl('-b', TEST_BRANCH)
 462         self.assertEqual(self._builder.count, self._total_builds)
 463         self.assertEqual(self._builder.fail, 0)
 464
 465     def testCount(self):
 466         """Test building a specific number of commitst"""
 467         self._RunControl('-b', TEST_BRANCH, '-c2')
 468         self.assertEqual(self._builder.count, 2 * len(boards))
 469         self.assertEqual(self._builder.fail, 0)
 470         # Each board has a mrproper, config, and then one make per commit
 471         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 472
 473     def testIncremental(self):
 474         """Test building a branch twice - the second time should do nothing"""
 475         self._RunControl('-b', TEST_BRANCH)
 476
 477         # Each board has a mrproper, config, and then one make per commit
 478         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 479         self._make_calls = 0
 480         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 481         self.assertEqual(self._make_calls, 0)
 482         self.assertEqual(self._builder.count, self._total_builds)
 483         self.assertEqual(self._builder.fail, 0)
 484
 485     def testForceBuild(self):
 486         """The -f flag should force a rebuild"""
 487         self._RunControl('-b', TEST_BRANCH)
 488         self._make_calls = 0
 489         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 490         # Each board has a mrproper, config, and then one make per commit
 491         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 492
 493     def testForceReconfigure(self):
 494         """The -f flag should force a rebuild"""
 495         self._RunControl('-b', TEST_BRANCH, '-C')
 496         # Each commit has a mrproper, config and make
 497         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 498
 499     def testErrors(self):
 500         """Test handling of build errors"""
 501         self._error['board2', 1] = 'fred\n'
 502         self._RunControl('-b', TEST_BRANCH)
 503         self.assertEqual(self._builder.count, self._total_builds)
 504         self.assertEqual(self._builder.fail, 1)
 505
 506         # Remove the error. This should have no effect since the commit will
 507         # not be rebuilt
 508         del self._error['board2', 1]
 509         self._make_calls = 0
 510         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 511         self.assertEqual(self._builder.count, self._total_builds)
 512         self.assertEqual(self._make_calls, 0)
 513         self.assertEqual(self._builder.fail, 1)
 514
 515         # Now use the -F flag to force rebuild of the bad commit
 516         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 517         self.assertEqual(self._builder.count, self._total_builds)
 518         self.assertEqual(self._builder.fail, 0)
 519         self.assertEqual(self._make_calls, 3)
 520
 521     def testBranchWithSlash(self):
 522         """Test building a branch with a '/' in the name"""
 523         self._test_branch = '/__dev/__testbranch'
 524         self._RunControl('-b', self._test_branch, clean_dir=False)
 525         self.assertEqual(self._builder.count, self._total_builds)
 526         self.assertEqual(self._builder.fail, 0)
 527
 528     def testBadOutputDir(self):
 529         """Test building with an output dir the same as out current dir"""
 530         self._test_branch = '/__dev/__testbranch'
 531         with self.assertRaises(SystemExit):
 532             self._RunControl('-b', self._test_branch, '-o', os.getcwd())
 533         with self.assertRaises(SystemExit):
 534             self._RunControl('-b', self._test_branch, '-o',
 535                              os.path.join(os.getcwd(), 'test'))