Merge branch 'buildman' of git://git.denx.de/u-boot-x86
[platform/kernel/u-boot.git] / tools / buildman / func_test.py
1 #
2 # Copyright (c) 2014 Google, Inc
3 #
4 # SPDX-License-Identifier:      GPL-2.0+
5 #
6
7 import os
8 import shutil
9 import sys
10 import tempfile
11 import unittest
12
13 import board
14 import bsettings
15 import cmdline
16 import command
17 import control
18 import gitutil
19 import terminal
20 import toolchain
21
22 settings_data = '''
23 # Buildman settings file
24
25 [toolchain]
26
27 [toolchain-alias]
28
29 [make-flags]
30 src=/home/sjg/c/src
31 chroot=/home/sjg/c/chroot
32 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
33 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
34 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
35 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
36 '''
37
38 boards = [
39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
40     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
41     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
42     ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
43     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
44 ]
45
46 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
47 39403bb patman: Use --no-pager' to stop git from forking a pager
48 db6e6f2 patman: Remove the -a option
49 f2ccf03 patman: Correct unit tests to run correctly
50 1d097f9 patman: Fix indentation in terminal.py
51 d073747 patman: Support the 'reverse' option for 'git log
52 """
53
54 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
55 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
56 Date:   Fri Aug 22 19:12:41 2014 +0900
57
58     buildman: refactor help message
59
60     "buildman [options]" is displayed by default.
61
62     Append the rest of help messages to parser.usage
63     instead of replacing it.
64
65     Besides, "-b <branch>" is not mandatory since commit fea5858e.
66     Drop it from the usage.
67
68     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
69 """,
70 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
71 Author: Simon Glass <sjg@chromium.org>
72 Date:   Thu Aug 14 16:48:25 2014 -0600
73
74     patman: Support the 'reverse' option for 'git log'
75
76     This option is currently not supported, but needs to be, for buildman to
77     operate as expected.
78
79     Series-changes: 7
80     - Add new patch to fix the 'reverse' bug
81
82     Series-version: 8
83
84     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
85     Reported-by: York Sun <yorksun@freescale.com>
86     Signed-off-by: Simon Glass <sjg@chromium.org>
87
88 """,
89 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
90 Author: Simon Glass <sjg@chromium.org>
91 Date:   Sat Aug 9 11:44:32 2014 -0600
92
93     patman: Fix indentation in terminal.py
94
95     This code came from a different project with 2-character indentation. Fix
96     it for U-Boot.
97
98     Series-changes: 6
99     - Add new patch to fix indentation in teminal.py
100
101     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
102     Signed-off-by: Simon Glass <sjg@chromium.org>
103
104 """,
105 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
106 Author: Simon Glass <sjg@chromium.org>
107 Date:   Sat Aug 9 11:08:24 2014 -0600
108
109     patman: Correct unit tests to run correctly
110
111     It seems that doctest behaves differently now, and some of the unit tests
112     do not run. Adjust the tests to work correctly.
113
114      ./tools/patman/patman --test
115     <unittest.result.TestResult run=10 errors=0 failures=0>
116
117     Series-changes: 6
118     - Add new patch to fix patman unit tests
119
120     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
121
122 """,
123 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
124 Author: Simon Glass <sjg@chromium.org>
125 Date:   Sat Aug 9 12:06:02 2014 -0600
126
127     patman: Remove the -a option
128
129     It seems that this is no longer needed, since checkpatch.pl will catch
130     whitespace problems in patches. Also the option is not widely used, so
131     it seems safe to just remove it.
132
133     Series-changes: 6
134     - Add new patch to remove patman's -a option
135
136     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
137     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
138
139 """,
140 """commit 39403bb4f838153028a6f21ca30bf100f3791133
141 Author: Simon Glass <sjg@chromium.org>
142 Date:   Thu Aug 14 21:50:52 2014 -0600
143
144     patman: Use --no-pager' to stop git from forking a pager
145
146 """,
147 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
148 Author: Simon Glass <sjg@chromium.org>
149 Date:   Fri Aug 22 15:57:39 2014 -0600
150
151     patman: Avoid changing the order of tags
152
153     patman collects tags that it sees in the commit and places them nicely
154     sorted at the end of the patch. However, this is not really necessary and
155     in fact is apparently not desirable.
156
157     Series-changes: 9
158     - Add new patch to avoid changing the order of tags
159
160     Series-version: 9
161
162     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
163     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
164 """]
165
166 TEST_BRANCH = '__testbranch'
167
168 class TestFunctional(unittest.TestCase):
169     """Functional test for buildman.
170
171     This aims to test from just below the invocation of buildman (parsing
172     of arguments) to 'make' and 'git' invocation. It is not a true
173     emd-to-end test, as it mocks git, make and the tool chain. But this
174     makes it easier to detect when the builder is doing the wrong thing,
175     since in many cases this test code will fail. For example, only a
176     very limited subset of 'git' arguments is supported - anything
177     unexpected will fail.
178     """
179     def setUp(self):
180         self._base_dir = tempfile.mkdtemp()
181         self._git_dir = os.path.join(self._base_dir, 'src')
182         self._buildman_pathname = sys.argv[0]
183         self._buildman_dir = os.path.dirname(sys.argv[0])
184         command.test_result = self._HandleCommand
185         self.setupToolchains()
186         self._toolchains.Add('arm-gcc', test=False)
187         self._toolchains.Add('powerpc-gcc', test=False)
188         bsettings.Setup(None)
189         bsettings.AddFile(settings_data)
190         self._boards = board.Boards()
191         for brd in boards:
192             self._boards.AddBoard(board.Board(*brd))
193
194         # Directories where the source been cloned
195         self._clone_dirs = []
196         self._commits = len(commit_shortlog.splitlines()) + 1
197         self._total_builds = self._commits * len(boards)
198
199         # Number of calls to make
200         self._make_calls = 0
201
202         # Map of [board, commit] to error messages
203         self._error = {}
204
205         self._test_branch = TEST_BRANCH
206
207         # Avoid sending any output and clear all terminal output
208         terminal.SetPrintTestMode()
209         terminal.GetPrintTestLines()
210
211     def tearDown(self):
212         shutil.rmtree(self._base_dir)
213
214     def setupToolchains(self):
215         self._toolchains = toolchain.Toolchains()
216         self._toolchains.Add('gcc', test=False)
217
218     def _RunBuildman(self, *args):
219         return command.RunPipe([[self._buildman_pathname] + list(args)],
220                 capture=True, capture_stderr=True)
221
222     def _RunControl(self, *args, **kwargs):
223         sys.argv = [sys.argv[0]] + list(args)
224         options, args = cmdline.ParseArgs()
225         result = control.DoBuildman(options, args, toolchains=self._toolchains,
226                 make_func=self._HandleMake, boards=self._boards,
227                 clean_dir=kwargs.get('clean_dir', True))
228         self._builder = control.builder
229         return result
230
231     def testFullHelp(self):
232         command.test_result = None
233         result = self._RunBuildman('-H')
234         help_file = os.path.join(self._buildman_dir, 'README')
235         self.assertEqual(len(result.stdout), os.path.getsize(help_file))
236         self.assertEqual(0, len(result.stderr))
237         self.assertEqual(0, result.return_code)
238
239     def testHelp(self):
240         command.test_result = None
241         result = self._RunBuildman('-h')
242         help_file = os.path.join(self._buildman_dir, 'README')
243         self.assertTrue(len(result.stdout) > 1000)
244         self.assertEqual(0, len(result.stderr))
245         self.assertEqual(0, result.return_code)
246
247     def testGitSetup(self):
248         """Test gitutils.Setup(), from outside the module itself"""
249         command.test_result = command.CommandResult(return_code=1)
250         gitutil.Setup()
251         self.assertEqual(gitutil.use_no_decorate, False)
252
253         command.test_result = command.CommandResult(return_code=0)
254         gitutil.Setup()
255         self.assertEqual(gitutil.use_no_decorate, True)
256
257     def _HandleCommandGitLog(self, args):
258         if '-n0' in args:
259             return command.CommandResult(return_code=0)
260         elif args[-1] == 'upstream/master..%s' % self._test_branch:
261             return command.CommandResult(return_code=0, stdout=commit_shortlog)
262         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
263             if args[-1] == self._test_branch:
264                 count = int(args[3][2:])
265                 return command.CommandResult(return_code=0,
266                                             stdout=''.join(commit_log[:count]))
267
268         # Not handled, so abort
269         print 'git log', args
270         sys.exit(1)
271
272     def _HandleCommandGitConfig(self, args):
273         config = args[0]
274         if config == 'sendemail.aliasesfile':
275             return command.CommandResult(return_code=0)
276         elif config.startswith('branch.badbranch'):
277             return command.CommandResult(return_code=1)
278         elif config == 'branch.%s.remote' % self._test_branch:
279             return command.CommandResult(return_code=0, stdout='upstream\n')
280         elif config == 'branch.%s.merge' % self._test_branch:
281             return command.CommandResult(return_code=0,
282                                          stdout='refs/heads/master\n')
283
284         # Not handled, so abort
285         print 'git config', args
286         sys.exit(1)
287
288     def _HandleCommandGit(self, in_args):
289         """Handle execution of a git command
290
291         This uses a hacked-up parser.
292
293         Args:
294             in_args: Arguments after 'git' from the command line
295         """
296         git_args = []           # Top-level arguments to git itself
297         sub_cmd = None          # Git sub-command selected
298         args = []               # Arguments to the git sub-command
299         for arg in in_args:
300             if sub_cmd:
301                 args.append(arg)
302             elif arg[0] == '-':
303                 git_args.append(arg)
304             else:
305                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
306                     git_args.append(arg)
307                 else:
308                     sub_cmd = arg
309         if sub_cmd == 'config':
310             return self._HandleCommandGitConfig(args)
311         elif sub_cmd == 'log':
312             return self._HandleCommandGitLog(args)
313         elif sub_cmd == 'clone':
314             return command.CommandResult(return_code=0)
315         elif sub_cmd == 'checkout':
316             return command.CommandResult(return_code=0)
317
318         # Not handled, so abort
319         print 'git', git_args, sub_cmd, args
320         sys.exit(1)
321
322     def _HandleCommandNm(self, args):
323         return command.CommandResult(return_code=0)
324
325     def _HandleCommandObjdump(self, args):
326         return command.CommandResult(return_code=0)
327
328     def _HandleCommandSize(self, args):
329         return command.CommandResult(return_code=0)
330
331     def _HandleCommand(self, **kwargs):
332         """Handle a command execution.
333
334         The command is in kwargs['pipe-list'], as a list of pipes, each a
335         list of commands. The command should be emulated as required for
336         testing purposes.
337
338         Returns:
339             A CommandResult object
340         """
341         pipe_list = kwargs['pipe_list']
342         wc = False
343         if len(pipe_list) != 1:
344             if pipe_list[1] == ['wc', '-l']:
345                 wc = True
346             else:
347                 print 'invalid pipe', kwargs
348                 sys.exit(1)
349         cmd = pipe_list[0][0]
350         args = pipe_list[0][1:]
351         result = None
352         if cmd == 'git':
353             result = self._HandleCommandGit(args)
354         elif cmd == './scripts/show-gnu-make':
355             return command.CommandResult(return_code=0, stdout='make')
356         elif cmd.endswith('nm'):
357             return self._HandleCommandNm(args)
358         elif cmd.endswith('objdump'):
359             return self._HandleCommandObjdump(args)
360         elif cmd.endswith( 'size'):
361             return self._HandleCommandSize(args)
362
363         if not result:
364             # Not handled, so abort
365             print 'unknown command', kwargs
366             sys.exit(1)
367
368         if wc:
369             result.stdout = len(result.stdout.splitlines())
370         return result
371
372     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
373         """Handle execution of 'make'
374
375         Args:
376             commit: Commit object that is being built
377             brd: Board object that is being built
378             stage: Stage that we are at (mrproper, config, build)
379             cwd: Directory where make should be run
380             args: Arguments to pass to make
381             kwargs: Arguments to pass to command.RunPipe()
382         """
383         self._make_calls += 1
384         if stage == 'mrproper':
385             return command.CommandResult(return_code=0)
386         elif stage == 'config':
387             return command.CommandResult(return_code=0,
388                     combined='Test configuration complete')
389         elif stage == 'build':
390             stderr = ''
391             if type(commit) is not str:
392                 stderr = self._error.get((brd.target, commit.sequence))
393             if stderr:
394                 return command.CommandResult(return_code=1, stderr=stderr)
395             return command.CommandResult(return_code=0)
396
397         # Not handled, so abort
398         print 'make', stage
399         sys.exit(1)
400
401     # Example function to print output lines
402     def print_lines(self, lines):
403         print len(lines)
404         for line in lines:
405             print line
406         #self.print_lines(terminal.GetPrintTestLines())
407
408     def testNoBoards(self):
409         """Test that buildman aborts when there are no boards"""
410         self._boards = board.Boards()
411         with self.assertRaises(SystemExit):
412             self._RunControl()
413
414     def testCurrentSource(self):
415         """Very simple test to invoke buildman on the current source"""
416         self.setupToolchains();
417         self._RunControl()
418         lines = terminal.GetPrintTestLines()
419         self.assertIn('Building current source for %d boards' % len(boards),
420                       lines[0].text)
421
422     def testBadBranch(self):
423         """Test that we can detect an invalid branch"""
424         with self.assertRaises(ValueError):
425             self._RunControl('-b', 'badbranch')
426
427     def testBadToolchain(self):
428         """Test that missing toolchains are detected"""
429         self.setupToolchains();
430         ret_code = self._RunControl('-b', TEST_BRANCH)
431         lines = terminal.GetPrintTestLines()
432
433         # Buildman always builds the upstream commit as well
434         self.assertIn('Building %d commits for %d boards' %
435                 (self._commits, len(boards)), lines[0].text)
436         self.assertEqual(self._builder.count, self._total_builds)
437
438         # Only sandbox should succeed, the others don't have toolchains
439         self.assertEqual(self._builder.fail,
440                          self._total_builds - self._commits)
441         self.assertEqual(ret_code, 128)
442
443         for commit in range(self._commits):
444             for board in self._boards.GetList():
445                 if board.arch != 'sandbox':
446                   errfile = self._builder.GetErrFile(commit, board.target)
447                   fd = open(errfile)
448                   self.assertEqual(fd.readlines(),
449                           ['No tool chain for %s\n' % board.arch])
450                   fd.close()
451
452     def testBranch(self):
453         """Test building a branch with all toolchains present"""
454         self._RunControl('-b', TEST_BRANCH)
455         self.assertEqual(self._builder.count, self._total_builds)
456         self.assertEqual(self._builder.fail, 0)
457
458     def testCount(self):
459         """Test building a specific number of commitst"""
460         self._RunControl('-b', TEST_BRANCH, '-c2')
461         self.assertEqual(self._builder.count, 2 * len(boards))
462         self.assertEqual(self._builder.fail, 0)
463         # Each board has a mrproper, config, and then one make per commit
464         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
465
466     def testIncremental(self):
467         """Test building a branch twice - the second time should do nothing"""
468         self._RunControl('-b', TEST_BRANCH)
469
470         # Each board has a mrproper, config, and then one make per commit
471         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
472         self._make_calls = 0
473         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
474         self.assertEqual(self._make_calls, 0)
475         self.assertEqual(self._builder.count, self._total_builds)
476         self.assertEqual(self._builder.fail, 0)
477
478     def testForceBuild(self):
479         """The -f flag should force a rebuild"""
480         self._RunControl('-b', TEST_BRANCH)
481         self._make_calls = 0
482         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
483         # Each board has a mrproper, config, and then one make per commit
484         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
485
486     def testForceReconfigure(self):
487         """The -f flag should force a rebuild"""
488         self._RunControl('-b', TEST_BRANCH, '-C')
489         # Each commit has a mrproper, config and make
490         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
491
492     def testErrors(self):
493         """Test handling of build errors"""
494         self._error['board2', 1] = 'fred\n'
495         self._RunControl('-b', TEST_BRANCH)
496         self.assertEqual(self._builder.count, self._total_builds)
497         self.assertEqual(self._builder.fail, 1)
498
499         # Remove the error. This should have no effect since the commit will
500         # not be rebuilt
501         del self._error['board2', 1]
502         self._make_calls = 0
503         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
504         self.assertEqual(self._builder.count, self._total_builds)
505         self.assertEqual(self._make_calls, 0)
506         self.assertEqual(self._builder.fail, 1)
507
508         # Now use the -F flag to force rebuild of the bad commit
509         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
510         self.assertEqual(self._builder.count, self._total_builds)
511         self.assertEqual(self._builder.fail, 0)
512         self.assertEqual(self._make_calls, 3)
513
514     def testBranchWithSlash(self):
515         """Test building a branch with a '/' in the name"""
516         self._test_branch = '/__dev/__testbranch'
517         self._RunControl('-b', self._test_branch, clean_dir=False)
518         self.assertEqual(self._builder.count, self._total_builds)
519         self.assertEqual(self._builder.fail, 0)