From bfb708ad9987ebddd2cd8f55bf4884e4f2305234 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 21 Feb 2023 12:40:29 -0700 Subject: [PATCH] buildman: Add a flag for reproducible builds This is quite a useful thing to use when building since it avoids small size changes between commits. Add a -r flag for it. Also undefine CONFIG_LOCALVERSION_AUTO since this appends the git hash to the version string, causing every build to be slightly different. Signed-off-by: Simon Glass --- doc/build/reproducible.rst | 2 ++ tools/buildman/builder.py | 4 +++- tools/buildman/builderthread.py | 2 ++ tools/buildman/buildman.rst | 7 ++++--- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 11 ++++++++++- tools/buildman/func_test.py | 40 +++++++++++++++++++++++++++++++++------- 7 files changed, 56 insertions(+), 12 deletions(-) diff --git a/doc/build/reproducible.rst b/doc/build/reproducible.rst index 5423080..8b030f4 100644 --- a/doc/build/reproducible.rst +++ b/doc/build/reproducible.rst @@ -23,3 +23,5 @@ This date is shown when we launch U-Boot: ./u-boot -T U-Boot 2023.01 (Jan 01 2023 - 00:00:00 +0000) + +The same effect can be obtained with buildman using the `-r` flag. diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 107086c..7b9be88 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -195,6 +195,7 @@ class Builder: don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs no_lto (bool): True to set the NO_LTO flag when building + reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds Private members: _base_board_dict: Last-summarised Dict of boards @@ -254,7 +255,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False, no_lto=False): + allow_missing=False, no_lto=False, reproducible_builds=False): """Create a new Builder object Args: @@ -334,6 +335,7 @@ class Builder: self.allow_missing = allow_missing self._ide = False self.no_lto = no_lto + self.reproducible_builds = reproducible_builds if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index dae3d4a..8b88c68 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -257,6 +257,8 @@ class BuilderThread(threading.Thread): args.append('BINMAN_ALLOW_MISSING=1') if self.builder.no_lto: args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 800b83a..c8b0db3 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the final binary. This information varies each time U-Boot is built. This causes various files to be rebuilt even if no source changes are made, which in turn requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. +be avoided by turning off the timestamp feature. This can be achieved using +the `-r` flag, which enables reproducible builds by setting +`SOURCE_DATE_EPOCH=0` when building. Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code:: - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + ./tools/buildman/buildman -Pr tegra Checking configuration diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 4090136..da7f1a9 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -97,6 +97,8 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-r', '--reproducible-builds', action='store_true', + help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_option('-R', '--regen-board-list', action='store_true', help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 13a462a..c3c5388 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, shutil.rmtree(output_dir) adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg) + # Drop LOCALVERSION_AUTO since it changes the version string on every commit + if options.reproducible_builds: + # If these are mentioned, leave the local version alone + if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: + print('Not dropping LOCALVERSION_AUTO for reproducible build') + else: + adjust_cfg['LOCALVERSION_AUTO'] = '~' + builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing, no_lto=options.no_lto) + allow_missing=allow_missing, no_lto=options.no_lto, + reproducible_builds=options.reproducible_builds) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6d15572..cf91c33 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase): kwargs: Arguments to pass to command.run_pipe() """ self._make_calls += 1 + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] if stage == 'mrproper': return command.CommandResult(return_code=0) elif stage == 'config': + fname = os.path.join(cwd or '', out_dir, '.config') + tools.write_file(fname, b'CONFIG_SOMETHING=1') return command.CommandResult(return_code=0, combined='Test configuration complete') elif stage == 'build': stderr = '' - out_dir = '' - for arg in args: - if arg.startswith('O='): - out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot') @@ -739,17 +741,41 @@ Some images are invalid''' cmd_fname = os.path.join(board0_dir, 'out-cmd') self.assertTrue(os.path.exists(cmd_fname)) data = tools.read_file(cmd_fname) - return data.splitlines() + + config_fname = os.path.join(board0_dir, '.config') + self.assertTrue(os.path.exists(config_fname)) + cfg_data = tools.read_file(config_fname) + + return data.splitlines(), cfg_data def testCmdFile(self): """Test that the -cmd-out file is produced""" - lines = self.check_command() + lines = self.check_command()[0] self.assertEqual(2, len(lines)) self.assertRegex(lines[0], b'make O=/.*board0_defconfig') self.assertRegex(lines[0], b'make O=/.*-s.*') def testNoLto(self): """Test that the --no-lto flag works""" - lines = self.check_command('-L') + lines = self.check_command('-L')[0] self.assertIn(b'NO_LTO=1', lines[0]) + def testReproducible(self): + """Test that the -r flag works""" + lines, cfg_data = self.check_command('-r') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +# CONFIG_LOCALVERSION_AUTO is not set +''', cfg_data) + + with test_util.capture_sys_output() as (stdout, stderr): + lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +CONFIG_LOCALVERSION=y +''', cfg_data) + self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue()) -- 2.7.4