tools/binman/ftest.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2016 Google, Inc
   3 # Written by Simon Glass <sjg@chromium.org>
   4 #
   5 # To run a single test, change to this directory, and:
   6 #
   7 #    python -m unittest func_test.TestFunctional.testHelp
   8
   9 import collections
  10 import gzip
  11 import hashlib
  12 from optparse import OptionParser
  13 import os
  14 import re
  15 import shutil
  16 import struct
  17 import sys
  18 import tempfile
  19 import unittest
  20 import unittest.mock
  21 import urllib.error
  22
  23 from binman import bintool
  24 from binman import cbfs_util
  25 from binman import cmdline
  26 from binman import comp_util
  27 from binman import control
  28 from binman import elf
  29 from binman import elf_test
  30 from binman import fip_util
  31 from binman import fmap_util
  32 from binman import state
  33 from dtoc import fdt
  34 from dtoc import fdt_util
  35 from binman.etype import fdtmap
  36 from binman.etype import image_header
  37 from binman.image import Image
  38 from patman import command
  39 from patman import test_util
  40 from patman import tools
  41 from patman import tout
  42
  43 # Contents of test files, corresponding to different entry types
  44 U_BOOT_DATA           = b'1234'
  45 U_BOOT_IMG_DATA       = b'img'
  46 U_BOOT_SPL_DATA       = b'56780123456789abcdefghi'
  47 U_BOOT_TPL_DATA       = b'tpl9876543210fedcbazyw'
  48 BLOB_DATA             = b'89'
  49 ME_DATA               = b'0abcd'
  50 VGA_DATA              = b'vga'
  51 U_BOOT_DTB_DATA       = b'udtb'
  52 U_BOOT_SPL_DTB_DATA   = b'spldtb'
  53 U_BOOT_TPL_DTB_DATA   = b'tpldtb'
  54 X86_START16_DATA      = b'start16'
  55 X86_START16_SPL_DATA  = b'start16spl'
  56 X86_START16_TPL_DATA  = b'start16tpl'
  57 X86_RESET16_DATA      = b'reset16'
  58 X86_RESET16_SPL_DATA  = b'reset16spl'
  59 X86_RESET16_TPL_DATA  = b'reset16tpl'
  60 PPC_MPC85XX_BR_DATA   = b'ppcmpc85xxbr'
  61 U_BOOT_NODTB_DATA     = b'nodtb with microcode pointer somewhere in here'
  62 U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here'
  63 U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here'
  64 U_BOOT_EXP_DATA       = U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA
  65 U_BOOT_SPL_EXP_DATA   = U_BOOT_SPL_NODTB_DATA + U_BOOT_SPL_DTB_DATA
  66 U_BOOT_TPL_EXP_DATA   = U_BOOT_TPL_NODTB_DATA + U_BOOT_TPL_DTB_DATA
  67 FSP_DATA              = b'fsp'
  68 CMC_DATA              = b'cmc'
  69 VBT_DATA              = b'vbt'
  70 MRC_DATA              = b'mrc'
  71 TEXT_DATA             = 'text'
  72 TEXT_DATA2            = 'text2'
  73 TEXT_DATA3            = 'text3'
  74 CROS_EC_RW_DATA       = b'ecrw'
  75 GBB_DATA              = b'gbbd'
  76 BMPBLK_DATA           = b'bmp'
  77 VBLOCK_DATA           = b'vblk'
  78 FILES_DATA            = (b"sorry I'm late\nOh, don't bother apologising, I'm " +
  79                          b"sorry you're alive\n")
  80 COMPRESS_DATA         = b'compress xxxxxxxxxxxxxxxxxxxxxx data'
  81 COMPRESS_DATA_BIG     = COMPRESS_DATA * 2
  82 REFCODE_DATA          = b'refcode'
  83 FSP_M_DATA            = b'fsp_m'
  84 FSP_S_DATA            = b'fsp_s'
  85 FSP_T_DATA            = b'fsp_t'
  86 ATF_BL31_DATA         = b'bl31'
  87 TEE_OS_DATA           = b'this is some tee OS data'
  88 ATF_BL2U_DATA         = b'bl2u'
  89 OPENSBI_DATA          = b'opensbi'
  90 SCP_DATA              = b'scp'
  91 TEST_FDT1_DATA        = b'fdt1'
  92 TEST_FDT2_DATA        = b'test-fdt2'
  93 ENV_DATA              = b'var1=1\nvar2="2"'
  94
  95 # Subdirectory of the input dir to use to put test FDTs
  96 TEST_FDT_SUBDIR       = 'fdts'
  97
  98 # The expected size for the device tree in some tests
  99 EXTRACT_DTB_SIZE = 0x3c9
 100
 101 # Properties expected to be in the device tree when update_dtb is used
 102 BASE_DTB_PROPS = ['offset', 'size', 'image-pos']
 103
 104 # Extra properties expected to be in the device tree when allow-repack is used
 105 REPACK_DTB_PROPS = ['orig-offset', 'orig-size']
 106
 107
 108 class TestFunctional(unittest.TestCase):
 109     """Functional tests for binman
 110
 111     Most of these use a sample .dts file to build an image and then check
 112     that it looks correct. The sample files are in the test/ subdirectory
 113     and are numbered.
 114
 115     For each entry type a very small test file is created using fixed
 116     string contents. This makes it easy to test that things look right, and
 117     debug problems.
 118
 119     In some cases a 'real' file must be used - these are also supplied in
 120     the test/ diurectory.
 121     """
 122     @classmethod
 123     def setUpClass(cls):
 124         global entry
 125         from binman import entry
 126
 127         # Handle the case where argv[0] is 'python'
 128         cls._binman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 129         cls._binman_pathname = os.path.join(cls._binman_dir, 'binman')
 130
 131         # Create a temporary directory for input files
 132         cls._indir = tempfile.mkdtemp(prefix='binmant.')
 133
 134         # Create some test files
 135         TestFunctional._MakeInputFile('u-boot.bin', U_BOOT_DATA)
 136         TestFunctional._MakeInputFile('u-boot.img', U_BOOT_IMG_DATA)
 137         TestFunctional._MakeInputFile('spl/u-boot-spl.bin', U_BOOT_SPL_DATA)
 138         TestFunctional._MakeInputFile('tpl/u-boot-tpl.bin', U_BOOT_TPL_DATA)
 139         TestFunctional._MakeInputFile('blobfile', BLOB_DATA)
 140         TestFunctional._MakeInputFile('me.bin', ME_DATA)
 141         TestFunctional._MakeInputFile('vga.bin', VGA_DATA)
 142         cls._ResetDtbs()
 143
 144         TestFunctional._MakeInputFile('u-boot-br.bin', PPC_MPC85XX_BR_DATA)
 145
 146         TestFunctional._MakeInputFile('u-boot-x86-start16.bin', X86_START16_DATA)
 147         TestFunctional._MakeInputFile('spl/u-boot-x86-start16-spl.bin',
 148                                       X86_START16_SPL_DATA)
 149         TestFunctional._MakeInputFile('tpl/u-boot-x86-start16-tpl.bin',
 150                                       X86_START16_TPL_DATA)
 151
 152         TestFunctional._MakeInputFile('u-boot-x86-reset16.bin',
 153                                       X86_RESET16_DATA)
 154         TestFunctional._MakeInputFile('spl/u-boot-x86-reset16-spl.bin',
 155                                       X86_RESET16_SPL_DATA)
 156         TestFunctional._MakeInputFile('tpl/u-boot-x86-reset16-tpl.bin',
 157                                       X86_RESET16_TPL_DATA)
 158
 159         TestFunctional._MakeInputFile('u-boot-nodtb.bin', U_BOOT_NODTB_DATA)
 160         TestFunctional._MakeInputFile('spl/u-boot-spl-nodtb.bin',
 161                                       U_BOOT_SPL_NODTB_DATA)
 162         TestFunctional._MakeInputFile('tpl/u-boot-tpl-nodtb.bin',
 163                                       U_BOOT_TPL_NODTB_DATA)
 164         TestFunctional._MakeInputFile('fsp.bin', FSP_DATA)
 165         TestFunctional._MakeInputFile('cmc.bin', CMC_DATA)
 166         TestFunctional._MakeInputFile('vbt.bin', VBT_DATA)
 167         TestFunctional._MakeInputFile('mrc.bin', MRC_DATA)
 168         TestFunctional._MakeInputFile('ecrw.bin', CROS_EC_RW_DATA)
 169         TestFunctional._MakeInputDir('devkeys')
 170         TestFunctional._MakeInputFile('bmpblk.bin', BMPBLK_DATA)
 171         TestFunctional._MakeInputFile('refcode.bin', REFCODE_DATA)
 172         TestFunctional._MakeInputFile('fsp_m.bin', FSP_M_DATA)
 173         TestFunctional._MakeInputFile('fsp_s.bin', FSP_S_DATA)
 174         TestFunctional._MakeInputFile('fsp_t.bin', FSP_T_DATA)
 175
 176         cls._elf_testdir = os.path.join(cls._indir, 'elftest')
 177         elf_test.BuildElfTestFiles(cls._elf_testdir)
 178
 179         # ELF file with a '_dt_ucode_base_size' symbol
 180         TestFunctional._MakeInputFile('u-boot',
 181             tools.read_file(cls.ElfTestFile('u_boot_ucode_ptr')))
 182
 183         # Intel flash descriptor file
 184         cls._SetupDescriptor()
 185
 186         shutil.copytree(cls.TestFile('files'),
 187                         os.path.join(cls._indir, 'files'))
 188
 189         TestFunctional._MakeInputFile('compress', COMPRESS_DATA)
 190         TestFunctional._MakeInputFile('compress_big', COMPRESS_DATA_BIG)
 191         TestFunctional._MakeInputFile('bl31.bin', ATF_BL31_DATA)
 192         TestFunctional._MakeInputFile('tee-pager.bin', TEE_OS_DATA)
 193         TestFunctional._MakeInputFile('bl2u.bin', ATF_BL2U_DATA)
 194         TestFunctional._MakeInputFile('fw_dynamic.bin', OPENSBI_DATA)
 195         TestFunctional._MakeInputFile('scp.bin', SCP_DATA)
 196
 197         # Add a few .dtb files for testing
 198         TestFunctional._MakeInputFile('%s/test-fdt1.dtb' % TEST_FDT_SUBDIR,
 199                                       TEST_FDT1_DATA)
 200         TestFunctional._MakeInputFile('%s/test-fdt2.dtb' % TEST_FDT_SUBDIR,
 201                                       TEST_FDT2_DATA)
 202
 203         TestFunctional._MakeInputFile('env.txt', ENV_DATA)
 204
 205         cls.have_lz4 = comp_util.HAVE_LZ4
 206
 207     @classmethod
 208     def tearDownClass(cls):
 209         """Remove the temporary input directory and its contents"""
 210         if cls.preserve_indir:
 211             print('Preserving input dir: %s' % cls._indir)
 212         else:
 213             if cls._indir:
 214                 shutil.rmtree(cls._indir)
 215         cls._indir = None
 216
 217     @classmethod
 218     def setup_test_args(cls, preserve_indir=False, preserve_outdirs=False,
 219                         toolpath=None, verbosity=None):
 220         """Accept arguments controlling test execution
 221
 222         Args:
 223             preserve_indir: Preserve the shared input directory used by all
 224                 tests in this class.
 225             preserve_outdir: Preserve the output directories used by tests. Each
 226                 test has its own, so this is normally only useful when running a
 227                 single test.
 228             toolpath: ist of paths to use for tools
 229         """
 230         cls.preserve_indir = preserve_indir
 231         cls.preserve_outdirs = preserve_outdirs
 232         cls.toolpath = toolpath
 233         cls.verbosity = verbosity
 234
 235     def _CheckLz4(self):
 236         if not self.have_lz4:
 237             self.skipTest('lz4 --no-frame-crc not available')
 238
 239     def _CleanupOutputDir(self):
 240         """Remove the temporary output directory"""
 241         if self.preserve_outdirs:
 242             print('Preserving output dir: %s' % tools.outdir)
 243         else:
 244             tools._finalise_for_test()
 245
 246     def setUp(self):
 247         # Enable this to turn on debugging output
 248         # tout.init(tout.DEBUG)
 249         command.test_result = None
 250
 251     def tearDown(self):
 252         """Remove the temporary output directory"""
 253         self._CleanupOutputDir()
 254
 255     def _SetupImageInTmpdir(self):
 256         """Set up the output image in a new temporary directory
 257
 258         This is used when an image has been generated in the output directory,
 259         but we want to run binman again. This will create a new output
 260         directory and fail to delete the original one.
 261
 262         This creates a new temporary directory, copies the image to it (with a
 263         new name) and removes the old output directory.
 264
 265         Returns:
 266             Tuple:
 267                 Temporary directory to use
 268                 New image filename
 269         """
 270         image_fname = tools.get_output_filename('image.bin')
 271         tmpdir = tempfile.mkdtemp(prefix='binman.')
 272         updated_fname = os.path.join(tmpdir, 'image-updated.bin')
 273         tools.write_file(updated_fname, tools.read_file(image_fname))
 274         self._CleanupOutputDir()
 275         return tmpdir, updated_fname
 276
 277     @classmethod
 278     def _ResetDtbs(cls):
 279         TestFunctional._MakeInputFile('u-boot.dtb', U_BOOT_DTB_DATA)
 280         TestFunctional._MakeInputFile('spl/u-boot-spl.dtb', U_BOOT_SPL_DTB_DATA)
 281         TestFunctional._MakeInputFile('tpl/u-boot-tpl.dtb', U_BOOT_TPL_DTB_DATA)
 282
 283     def _RunBinman(self, *args, **kwargs):
 284         """Run binman using the command line
 285
 286         Args:
 287             Arguments to pass, as a list of strings
 288             kwargs: Arguments to pass to Command.RunPipe()
 289         """
 290         result = command.run_pipe([[self._binman_pathname] + list(args)],
 291                 capture=True, capture_stderr=True, raise_on_error=False)
 292         if result.return_code and kwargs.get('raise_on_error', True):
 293             raise Exception("Error running '%s': %s" % (' '.join(args),
 294                             result.stdout + result.stderr))
 295         return result
 296
 297     def _DoBinman(self, *argv):
 298         """Run binman using directly (in the same process)
 299
 300         Args:
 301             Arguments to pass, as a list of strings
 302         Returns:
 303             Return value (0 for success)
 304         """
 305         argv = list(argv)
 306         args = cmdline.ParseArgs(argv)
 307         args.pager = 'binman-invalid-pager'
 308         args.build_dir = self._indir
 309
 310         # For testing, you can force an increase in verbosity here
 311         # args.verbosity = tout.DEBUG
 312         return control.Binman(args)
 313
 314     def _DoTestFile(self, fname, debug=False, map=False, update_dtb=False,
 315                     entry_args=None, images=None, use_real_dtb=False,
 316                     use_expanded=False, verbosity=None, allow_missing=False,
 317                     allow_fake_blobs=False, extra_indirs=None, threads=None,
 318                     test_section_timeout=False, update_fdt_in_elf=None,
 319                     force_missing_bintools=''):
 320         """Run binman with a given test file
 321
 322         Args:
 323             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 324             debug: True to enable debugging output
 325             map: True to output map files for the images
 326             update_dtb: Update the offset and size of each entry in the device
 327                 tree before packing it into the image
 328             entry_args: Dict of entry args to supply to binman
 329                 key: arg name
 330                 value: value of that arg
 331             images: List of image names to build
 332             use_real_dtb: True to use the test file as the contents of
 333                 the u-boot-dtb entry. Normally this is not needed and the
 334                 test contents (the U_BOOT_DTB_DATA string) can be used.
 335                 But in some test we need the real contents.
 336             use_expanded: True to use expanded entries where available, e.g.
 337                 'u-boot-expanded' instead of 'u-boot'
 338             verbosity: Verbosity level to use (0-3, None=don't set it)
 339             allow_missing: Set the '--allow-missing' flag so that missing
 340                 external binaries just produce a warning instead of an error
 341             allow_fake_blobs: Set the '--fake-ext-blobs' flag
 342             extra_indirs: Extra input directories to add using -I
 343             threads: Number of threads to use (None for default, 0 for
 344                 single-threaded)
 345             test_section_timeout: True to force the first time to timeout, as
 346                 used in testThreadTimeout()
 347             update_fdt_in_elf: Value to pass with --update-fdt-in-elf=xxx
 348             force_missing_tools (str): comma-separated list of bintools to
 349                 regard as missing
 350
 351         Returns:
 352             int return code, 0 on success
 353         """
 354         args = []
 355         if debug:
 356             args.append('-D')
 357         if verbosity is not None:
 358             args.append('-v%d' % verbosity)
 359         elif self.verbosity:
 360             args.append('-v%d' % self.verbosity)
 361         if self.toolpath:
 362             for path in self.toolpath:
 363                 args += ['--toolpath', path]
 364         if threads is not None:
 365             args.append('-T%d' % threads)
 366         if test_section_timeout:
 367             args.append('--test-section-timeout')
 368         args += ['build', '-p', '-I', self._indir, '-d', self.TestFile(fname)]
 369         if map:
 370             args.append('-m')
 371         if update_dtb:
 372             args.append('-u')
 373         if not use_real_dtb:
 374             args.append('--fake-dtb')
 375         if not use_expanded:
 376             args.append('--no-expanded')
 377         if entry_args:
 378             for arg, value in entry_args.items():
 379                 args.append('-a%s=%s' % (arg, value))
 380         if allow_missing:
 381             args.append('-M')
 382         if allow_fake_blobs:
 383             args.append('--fake-ext-blobs')
 384         if force_missing_bintools:
 385             args += ['--force-missing-bintools', force_missing_bintools]
 386         if update_fdt_in_elf:
 387             args += ['--update-fdt-in-elf', update_fdt_in_elf]
 388         if images:
 389             for image in images:
 390                 args += ['-i', image]
 391         if extra_indirs:
 392             for indir in extra_indirs:
 393                 args += ['-I', indir]
 394         return self._DoBinman(*args)
 395
 396     def _SetupDtb(self, fname, outfile='u-boot.dtb'):
 397         """Set up a new test device-tree file
 398
 399         The given file is compiled and set up as the device tree to be used
 400         for ths test.
 401
 402         Args:
 403             fname: Filename of .dts file to read
 404             outfile: Output filename for compiled device-tree binary
 405
 406         Returns:
 407             Contents of device-tree binary
 408         """
 409         tmpdir = tempfile.mkdtemp(prefix='binmant.')
 410         dtb = fdt_util.EnsureCompiled(self.TestFile(fname), tmpdir)
 411         with open(dtb, 'rb') as fd:
 412             data = fd.read()
 413             TestFunctional._MakeInputFile(outfile, data)
 414         shutil.rmtree(tmpdir)
 415         return data
 416
 417     def _GetDtbContentsForSplTpl(self, dtb_data, name):
 418         """Create a version of the main DTB for SPL or SPL
 419
 420         For testing we don't actually have different versions of the DTB. With
 421         U-Boot we normally run fdtgrep to remove unwanted nodes, but for tests
 422         we don't normally have any unwanted nodes.
 423
 424         We still want the DTBs for SPL and TPL to be different though, since
 425         otherwise it is confusing to know which one we are looking at. So add
 426         an 'spl' or 'tpl' property to the top-level node.
 427
 428         Args:
 429             dtb_data: dtb data to modify (this should be a value devicetree)
 430             name: Name of a new property to add
 431
 432         Returns:
 433             New dtb data with the property added
 434         """
 435         dtb = fdt.Fdt.FromData(dtb_data)
 436         dtb.Scan()
 437         dtb.GetNode('/binman').AddZeroProp(name)
 438         dtb.Sync(auto_resize=True)
 439         dtb.Pack()
 440         return dtb.GetContents()
 441
 442     def _DoReadFileDtb(self, fname, use_real_dtb=False, use_expanded=False,
 443                        map=False, update_dtb=False, entry_args=None,
 444                        reset_dtbs=True, extra_indirs=None, threads=None):
 445         """Run binman and return the resulting image
 446
 447         This runs binman with a given test file and then reads the resulting
 448         output file. It is a shortcut function since most tests need to do
 449         these steps.
 450
 451         Raises an assertion failure if binman returns a non-zero exit code.
 452
 453         Args:
 454             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 455             use_real_dtb: True to use the test file as the contents of
 456                 the u-boot-dtb entry. Normally this is not needed and the
 457                 test contents (the U_BOOT_DTB_DATA string) can be used.
 458                 But in some test we need the real contents.
 459             use_expanded: True to use expanded entries where available, e.g.
 460                 'u-boot-expanded' instead of 'u-boot'
 461             map: True to output map files for the images
 462             update_dtb: Update the offset and size of each entry in the device
 463                 tree before packing it into the image
 464             entry_args: Dict of entry args to supply to binman
 465                 key: arg name
 466                 value: value of that arg
 467             reset_dtbs: With use_real_dtb the test dtb is overwritten by this
 468                 function. If reset_dtbs is True, then the original test dtb
 469                 is written back before this function finishes
 470             extra_indirs: Extra input directories to add using -I
 471             threads: Number of threads to use (None for default, 0 for
 472                 single-threaded)
 473
 474         Returns:
 475             Tuple:
 476                 Resulting image contents
 477                 Device tree contents
 478                 Map data showing contents of image (or None if none)
 479                 Output device tree binary filename ('u-boot.dtb' path)
 480         """
 481         dtb_data = None
 482         # Use the compiled test file as the u-boot-dtb input
 483         if use_real_dtb:
 484             dtb_data = self._SetupDtb(fname)
 485
 486             # For testing purposes, make a copy of the DT for SPL and TPL. Add
 487             # a node indicating which it is, so aid verification.
 488             for name in ['spl', 'tpl']:
 489                 dtb_fname = '%s/u-boot-%s.dtb' % (name, name)
 490                 outfile = os.path.join(self._indir, dtb_fname)
 491                 TestFunctional._MakeInputFile(dtb_fname,
 492                         self._GetDtbContentsForSplTpl(dtb_data, name))
 493
 494         try:
 495             retcode = self._DoTestFile(fname, map=map, update_dtb=update_dtb,
 496                     entry_args=entry_args, use_real_dtb=use_real_dtb,
 497                     use_expanded=use_expanded, extra_indirs=extra_indirs,
 498                     threads=threads)
 499             self.assertEqual(0, retcode)
 500             out_dtb_fname = tools.get_output_filename('u-boot.dtb.out')
 501
 502             # Find the (only) image, read it and return its contents
 503             image = control.images['image']
 504             image_fname = tools.get_output_filename('image.bin')
 505             self.assertTrue(os.path.exists(image_fname))
 506             if map:
 507                 map_fname = tools.get_output_filename('image.map')
 508                 with open(map_fname) as fd:
 509                     map_data = fd.read()
 510             else:
 511                 map_data = None
 512             with open(image_fname, 'rb') as fd:
 513                 return fd.read(), dtb_data, map_data, out_dtb_fname
 514         finally:
 515             # Put the test file back
 516             if reset_dtbs and use_real_dtb:
 517                 self._ResetDtbs()
 518
 519     def _DoReadFileRealDtb(self, fname):
 520         """Run binman with a real .dtb file and return the resulting data
 521
 522         Args:
 523             fname: DT source filename to use (e.g. 082_fdt_update_all.dts)
 524
 525         Returns:
 526             Resulting image contents
 527         """
 528         return self._DoReadFileDtb(fname, use_real_dtb=True, update_dtb=True)[0]
 529
 530     def _DoReadFile(self, fname, use_real_dtb=False):
 531         """Helper function which discards the device-tree binary
 532
 533         Args:
 534             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 535             use_real_dtb: True to use the test file as the contents of
 536                 the u-boot-dtb entry. Normally this is not needed and the
 537                 test contents (the U_BOOT_DTB_DATA string) can be used.
 538                 But in some test we need the real contents.
 539
 540         Returns:
 541             Resulting image contents
 542         """
 543         return self._DoReadFileDtb(fname, use_real_dtb)[0]
 544
 545     @classmethod
 546     def _MakeInputFile(cls, fname, contents):
 547         """Create a new test input file, creating directories as needed
 548
 549         Args:
 550             fname: Filename to create
 551             contents: File contents to write in to the file
 552         Returns:
 553             Full pathname of file created
 554         """
 555         pathname = os.path.join(cls._indir, fname)
 556         dirname = os.path.dirname(pathname)
 557         if dirname and not os.path.exists(dirname):
 558             os.makedirs(dirname)
 559         with open(pathname, 'wb') as fd:
 560             fd.write(contents)
 561         return pathname
 562
 563     @classmethod
 564     def _MakeInputDir(cls, dirname):
 565         """Create a new test input directory, creating directories as needed
 566
 567         Args:
 568             dirname: Directory name to create
 569
 570         Returns:
 571             Full pathname of directory created
 572         """
 573         pathname = os.path.join(cls._indir, dirname)
 574         if not os.path.exists(pathname):
 575             os.makedirs(pathname)
 576         return pathname
 577
 578     @classmethod
 579     def _SetupSplElf(cls, src_fname='bss_data'):
 580         """Set up an ELF file with a '_dt_ucode_base_size' symbol
 581
 582         Args:
 583             Filename of ELF file to use as SPL
 584         """
 585         TestFunctional._MakeInputFile('spl/u-boot-spl',
 586             tools.read_file(cls.ElfTestFile(src_fname)))
 587
 588     @classmethod
 589     def _SetupTplElf(cls, src_fname='bss_data'):
 590         """Set up an ELF file with a '_dt_ucode_base_size' symbol
 591
 592         Args:
 593             Filename of ELF file to use as TPL
 594         """
 595         TestFunctional._MakeInputFile('tpl/u-boot-tpl',
 596             tools.read_file(cls.ElfTestFile(src_fname)))
 597
 598     @classmethod
 599     def _SetupDescriptor(cls):
 600         with open(cls.TestFile('descriptor.bin'), 'rb') as fd:
 601             TestFunctional._MakeInputFile('descriptor.bin', fd.read())
 602
 603     @classmethod
 604     def TestFile(cls, fname):
 605         return os.path.join(cls._binman_dir, 'test', fname)
 606
 607     @classmethod
 608     def ElfTestFile(cls, fname):
 609         return os.path.join(cls._elf_testdir, fname)
 610
 611     def AssertInList(self, grep_list, target):
 612         """Assert that at least one of a list of things is in a target
 613
 614         Args:
 615             grep_list: List of strings to check
 616             target: Target string
 617         """
 618         for grep in grep_list:
 619             if grep in target:
 620                 return
 621         self.fail("Error: '%s' not found in '%s'" % (grep_list, target))
 622
 623     def CheckNoGaps(self, entries):
 624         """Check that all entries fit together without gaps
 625
 626         Args:
 627             entries: List of entries to check
 628         """
 629         offset = 0
 630         for entry in entries.values():
 631             self.assertEqual(offset, entry.offset)
 632             offset += entry.size
 633
 634     def GetFdtLen(self, dtb):
 635         """Get the totalsize field from a device-tree binary
 636
 637         Args:
 638             dtb: Device-tree binary contents
 639
 640         Returns:
 641             Total size of device-tree binary, from the header
 642         """
 643         return struct.unpack('>L', dtb[4:8])[0]
 644
 645     def _GetPropTree(self, dtb, prop_names, prefix='/binman/'):
 646         def AddNode(node, path):
 647             if node.name != '/':
 648                 path += '/' + node.name
 649             for prop in node.props.values():
 650                 if prop.name in prop_names:
 651                     prop_path = path + ':' + prop.name
 652                     tree[prop_path[len(prefix):]] = fdt_util.fdt32_to_cpu(
 653                         prop.value)
 654             for subnode in node.subnodes:
 655                 AddNode(subnode, path)
 656
 657         tree = {}
 658         AddNode(dtb.GetRoot(), '')
 659         return tree
 660
 661     def testRun(self):
 662         """Test a basic run with valid args"""
 663         result = self._RunBinman('-h')
 664
 665     def testFullHelp(self):
 666         """Test that the full help is displayed with -H"""
 667         result = self._RunBinman('-H')
 668         help_file = os.path.join(self._binman_dir, 'README.rst')
 669         # Remove possible extraneous strings
 670         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 671         gothelp = result.stdout.replace(extra, '')
 672         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 673         self.assertEqual(0, len(result.stderr))
 674         self.assertEqual(0, result.return_code)
 675
 676     def testFullHelpInternal(self):
 677         """Test that the full help is displayed with -H"""
 678         try:
 679             command.test_result = command.CommandResult()
 680             result = self._DoBinman('-H')
 681             help_file = os.path.join(self._binman_dir, 'README.rst')
 682         finally:
 683             command.test_result = None
 684
 685     def testHelp(self):
 686         """Test that the basic help is displayed with -h"""
 687         result = self._RunBinman('-h')
 688         self.assertTrue(len(result.stdout) > 200)
 689         self.assertEqual(0, len(result.stderr))
 690         self.assertEqual(0, result.return_code)
 691
 692     def testBoard(self):
 693         """Test that we can run it with a specific board"""
 694         self._SetupDtb('005_simple.dts', 'sandbox/u-boot.dtb')
 695         TestFunctional._MakeInputFile('sandbox/u-boot.bin', U_BOOT_DATA)
 696         result = self._DoBinman('build', '-n', '-b', 'sandbox')
 697         self.assertEqual(0, result)
 698
 699     def testNeedBoard(self):
 700         """Test that we get an error when no board ius supplied"""
 701         with self.assertRaises(ValueError) as e:
 702             result = self._DoBinman('build')
 703         self.assertIn("Must provide a board to process (use -b <board>)",
 704                 str(e.exception))
 705
 706     def testMissingDt(self):
 707         """Test that an invalid device-tree file generates an error"""
 708         with self.assertRaises(Exception) as e:
 709             self._RunBinman('build', '-d', 'missing_file')
 710         # We get one error from libfdt, and a different one from fdtget.
 711         self.AssertInList(["Couldn't open blob from 'missing_file'",
 712                            'No such file or directory'], str(e.exception))
 713
 714     def testBrokenDt(self):
 715         """Test that an invalid device-tree source file generates an error
 716
 717         Since this is a source file it should be compiled and the error
 718         will come from the device-tree compiler (dtc).
 719         """
 720         with self.assertRaises(Exception) as e:
 721             self._RunBinman('build', '-d', self.TestFile('001_invalid.dts'))
 722         self.assertIn("FATAL ERROR: Unable to parse input tree",
 723                 str(e.exception))
 724
 725     def testMissingNode(self):
 726         """Test that a device tree without a 'binman' node generates an error"""
 727         with self.assertRaises(Exception) as e:
 728             self._DoBinman('build', '-d', self.TestFile('002_missing_node.dts'))
 729         self.assertIn("does not have a 'binman' node", str(e.exception))
 730
 731     def testEmpty(self):
 732         """Test that an empty binman node works OK (i.e. does nothing)"""
 733         result = self._RunBinman('build', '-d', self.TestFile('003_empty.dts'))
 734         self.assertEqual(0, len(result.stderr))
 735         self.assertEqual(0, result.return_code)
 736
 737     def testInvalidEntry(self):
 738         """Test that an invalid entry is flagged"""
 739         with self.assertRaises(Exception) as e:
 740             result = self._RunBinman('build', '-d',
 741                                      self.TestFile('004_invalid_entry.dts'))
 742         self.assertIn("Unknown entry type 'not-a-valid-type' in node "
 743                 "'/binman/not-a-valid-type'", str(e.exception))
 744
 745     def testSimple(self):
 746         """Test a simple binman with a single file"""
 747         data = self._DoReadFile('005_simple.dts')
 748         self.assertEqual(U_BOOT_DATA, data)
 749
 750     def testSimpleDebug(self):
 751         """Test a simple binman run with debugging enabled"""
 752         self._DoTestFile('005_simple.dts', debug=True)
 753
 754     def testDual(self):
 755         """Test that we can handle creating two images
 756
 757         This also tests image padding.
 758         """
 759         retcode = self._DoTestFile('006_dual_image.dts')
 760         self.assertEqual(0, retcode)
 761
 762         image = control.images['image1']
 763         self.assertEqual(len(U_BOOT_DATA), image.size)
 764         fname = tools.get_output_filename('image1.bin')
 765         self.assertTrue(os.path.exists(fname))
 766         with open(fname, 'rb') as fd:
 767             data = fd.read()
 768             self.assertEqual(U_BOOT_DATA, data)
 769
 770         image = control.images['image2']
 771         self.assertEqual(3 + len(U_BOOT_DATA) + 5, image.size)
 772         fname = tools.get_output_filename('image2.bin')
 773         self.assertTrue(os.path.exists(fname))
 774         with open(fname, 'rb') as fd:
 775             data = fd.read()
 776             self.assertEqual(U_BOOT_DATA, data[3:7])
 777             self.assertEqual(tools.get_bytes(0, 3), data[:3])
 778             self.assertEqual(tools.get_bytes(0, 5), data[7:])
 779
 780     def testBadAlign(self):
 781         """Test that an invalid alignment value is detected"""
 782         with self.assertRaises(ValueError) as e:
 783             self._DoTestFile('007_bad_align.dts')
 784         self.assertIn("Node '/binman/u-boot': Alignment 23 must be a power "
 785                       "of two", str(e.exception))
 786
 787     def testPackSimple(self):
 788         """Test that packing works as expected"""
 789         retcode = self._DoTestFile('008_pack.dts')
 790         self.assertEqual(0, retcode)
 791         self.assertIn('image', control.images)
 792         image = control.images['image']
 793         entries = image.GetEntries()
 794         self.assertEqual(5, len(entries))
 795
 796         # First u-boot
 797         self.assertIn('u-boot', entries)
 798         entry = entries['u-boot']
 799         self.assertEqual(0, entry.offset)
 800         self.assertEqual(len(U_BOOT_DATA), entry.size)
 801
 802         # Second u-boot, aligned to 16-byte boundary
 803         self.assertIn('u-boot-align', entries)
 804         entry = entries['u-boot-align']
 805         self.assertEqual(16, entry.offset)
 806         self.assertEqual(len(U_BOOT_DATA), entry.size)
 807
 808         # Third u-boot, size 23 bytes
 809         self.assertIn('u-boot-size', entries)
 810         entry = entries['u-boot-size']
 811         self.assertEqual(20, entry.offset)
 812         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
 813         self.assertEqual(23, entry.size)
 814
 815         # Fourth u-boot, placed immediate after the above
 816         self.assertIn('u-boot-next', entries)
 817         entry = entries['u-boot-next']
 818         self.assertEqual(43, entry.offset)
 819         self.assertEqual(len(U_BOOT_DATA), entry.size)
 820
 821         # Fifth u-boot, placed at a fixed offset
 822         self.assertIn('u-boot-fixed', entries)
 823         entry = entries['u-boot-fixed']
 824         self.assertEqual(61, entry.offset)
 825         self.assertEqual(len(U_BOOT_DATA), entry.size)
 826
 827         self.assertEqual(65, image.size)
 828
 829     def testPackExtra(self):
 830         """Test that extra packing feature works as expected"""
 831         data, _, _, out_dtb_fname = self._DoReadFileDtb('009_pack_extra.dts',
 832                                                         update_dtb=True)
 833
 834         self.assertIn('image', control.images)
 835         image = control.images['image']
 836         entries = image.GetEntries()
 837         self.assertEqual(5, len(entries))
 838
 839         # First u-boot with padding before and after
 840         self.assertIn('u-boot', entries)
 841         entry = entries['u-boot']
 842         self.assertEqual(0, entry.offset)
 843         self.assertEqual(3, entry.pad_before)
 844         self.assertEqual(3 + 5 + len(U_BOOT_DATA), entry.size)
 845         self.assertEqual(U_BOOT_DATA, entry.data)
 846         self.assertEqual(tools.get_bytes(0, 3) + U_BOOT_DATA +
 847                          tools.get_bytes(0, 5), data[:entry.size])
 848         pos = entry.size
 849
 850         # Second u-boot has an aligned size, but it has no effect
 851         self.assertIn('u-boot-align-size-nop', entries)
 852         entry = entries['u-boot-align-size-nop']
 853         self.assertEqual(pos, entry.offset)
 854         self.assertEqual(len(U_BOOT_DATA), entry.size)
 855         self.assertEqual(U_BOOT_DATA, entry.data)
 856         self.assertEqual(U_BOOT_DATA, data[pos:pos + entry.size])
 857         pos += entry.size
 858
 859         # Third u-boot has an aligned size too
 860         self.assertIn('u-boot-align-size', entries)
 861         entry = entries['u-boot-align-size']
 862         self.assertEqual(pos, entry.offset)
 863         self.assertEqual(32, entry.size)
 864         self.assertEqual(U_BOOT_DATA, entry.data)
 865         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 32 - len(U_BOOT_DATA)),
 866                          data[pos:pos + entry.size])
 867         pos += entry.size
 868
 869         # Fourth u-boot has an aligned end
 870         self.assertIn('u-boot-align-end', entries)
 871         entry = entries['u-boot-align-end']
 872         self.assertEqual(48, entry.offset)
 873         self.assertEqual(16, entry.size)
 874         self.assertEqual(U_BOOT_DATA, entry.data[:len(U_BOOT_DATA)])
 875         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 16 - len(U_BOOT_DATA)),
 876                          data[pos:pos + entry.size])
 877         pos += entry.size
 878
 879         # Fifth u-boot immediately afterwards
 880         self.assertIn('u-boot-align-both', entries)
 881         entry = entries['u-boot-align-both']
 882         self.assertEqual(64, entry.offset)
 883         self.assertEqual(64, entry.size)
 884         self.assertEqual(U_BOOT_DATA, entry.data[:len(U_BOOT_DATA)])
 885         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 64 - len(U_BOOT_DATA)),
 886                          data[pos:pos + entry.size])
 887
 888         self.CheckNoGaps(entries)
 889         self.assertEqual(128, image.size)
 890
 891         dtb = fdt.Fdt(out_dtb_fname)
 892         dtb.Scan()
 893         props = self._GetPropTree(dtb, ['size', 'offset', 'image-pos'])
 894         expected = {
 895             'image-pos': 0,
 896             'offset': 0,
 897             'size': 128,
 898
 899             'u-boot:image-pos': 0,
 900             'u-boot:offset': 0,
 901             'u-boot:size': 3 + 5 + len(U_BOOT_DATA),
 902
 903             'u-boot-align-size-nop:image-pos': 12,
 904             'u-boot-align-size-nop:offset': 12,
 905             'u-boot-align-size-nop:size': 4,
 906
 907             'u-boot-align-size:image-pos': 16,
 908             'u-boot-align-size:offset': 16,
 909             'u-boot-align-size:size': 32,
 910
 911             'u-boot-align-end:image-pos': 48,
 912             'u-boot-align-end:offset': 48,
 913             'u-boot-align-end:size': 16,
 914
 915             'u-boot-align-both:image-pos': 64,
 916             'u-boot-align-both:offset': 64,
 917             'u-boot-align-both:size': 64,
 918             }
 919         self.assertEqual(expected, props)
 920
 921     def testPackAlignPowerOf2(self):
 922         """Test that invalid entry alignment is detected"""
 923         with self.assertRaises(ValueError) as e:
 924             self._DoTestFile('010_pack_align_power2.dts')
 925         self.assertIn("Node '/binman/u-boot': Alignment 5 must be a power "
 926                       "of two", str(e.exception))
 927
 928     def testPackAlignSizePowerOf2(self):
 929         """Test that invalid entry size alignment is detected"""
 930         with self.assertRaises(ValueError) as e:
 931             self._DoTestFile('011_pack_align_size_power2.dts')
 932         self.assertIn("Node '/binman/u-boot': Alignment size 55 must be a "
 933                       "power of two", str(e.exception))
 934
 935     def testPackInvalidAlign(self):
 936         """Test detection of an offset that does not match its alignment"""
 937         with self.assertRaises(ValueError) as e:
 938             self._DoTestFile('012_pack_inv_align.dts')
 939         self.assertIn("Node '/binman/u-boot': Offset 0x5 (5) does not match "
 940                       "align 0x4 (4)", str(e.exception))
 941
 942     def testPackInvalidSizeAlign(self):
 943         """Test that invalid entry size alignment is detected"""
 944         with self.assertRaises(ValueError) as e:
 945             self._DoTestFile('013_pack_inv_size_align.dts')
 946         self.assertIn("Node '/binman/u-boot': Size 0x5 (5) does not match "
 947                       "align-size 0x4 (4)", str(e.exception))
 948
 949     def testPackOverlap(self):
 950         """Test that overlapping regions are detected"""
 951         with self.assertRaises(ValueError) as e:
 952             self._DoTestFile('014_pack_overlap.dts')
 953         self.assertIn("Node '/binman/u-boot-align': Offset 0x3 (3) overlaps "
 954                       "with previous entry '/binman/u-boot' ending at 0x4 (4)",
 955                       str(e.exception))
 956
 957     def testPackEntryOverflow(self):
 958         """Test that entries that overflow their size are detected"""
 959         with self.assertRaises(ValueError) as e:
 960             self._DoTestFile('015_pack_overflow.dts')
 961         self.assertIn("Node '/binman/u-boot': Entry contents size is 0x4 (4) "
 962                       "but entry size is 0x3 (3)", str(e.exception))
 963
 964     def testPackImageOverflow(self):
 965         """Test that entries which overflow the image size are detected"""
 966         with self.assertRaises(ValueError) as e:
 967             self._DoTestFile('016_pack_image_overflow.dts')
 968         self.assertIn("Section '/binman': contents size 0x4 (4) exceeds section "
 969                       "size 0x3 (3)", str(e.exception))
 970
 971     def testPackImageSize(self):
 972         """Test that the image size can be set"""
 973         retcode = self._DoTestFile('017_pack_image_size.dts')
 974         self.assertEqual(0, retcode)
 975         self.assertIn('image', control.images)
 976         image = control.images['image']
 977         self.assertEqual(7, image.size)
 978
 979     def testPackImageSizeAlign(self):
 980         """Test that image size alignemnt works as expected"""
 981         retcode = self._DoTestFile('018_pack_image_align.dts')
 982         self.assertEqual(0, retcode)
 983         self.assertIn('image', control.images)
 984         image = control.images['image']
 985         self.assertEqual(16, image.size)
 986
 987     def testPackInvalidImageAlign(self):
 988         """Test that invalid image alignment is detected"""
 989         with self.assertRaises(ValueError) as e:
 990             self._DoTestFile('019_pack_inv_image_align.dts')
 991         self.assertIn("Section '/binman': Size 0x7 (7) does not match "
 992                       "align-size 0x8 (8)", str(e.exception))
 993
 994     def testPackAlignPowerOf2Inv(self):
 995         """Test that invalid image alignment is detected"""
 996         with self.assertRaises(ValueError) as e:
 997             self._DoTestFile('020_pack_inv_image_align_power2.dts')
 998         self.assertIn("Image '/binman': Alignment size 131 must be a power of "
 999                       "two", str(e.exception))
1000
1001     def testImagePadByte(self):
1002         """Test that the image pad byte can be specified"""
1003         self._SetupSplElf()
1004         data = self._DoReadFile('021_image_pad.dts')
1005         self.assertEqual(U_BOOT_SPL_DATA + tools.get_bytes(0xff, 1) +
1006                          U_BOOT_DATA, data)
1007
1008     def testImageName(self):
1009         """Test that image files can be named"""
1010         retcode = self._DoTestFile('022_image_name.dts')
1011         self.assertEqual(0, retcode)
1012         image = control.images['image1']
1013         fname = tools.get_output_filename('test-name')
1014         self.assertTrue(os.path.exists(fname))
1015
1016         image = control.images['image2']
1017         fname = tools.get_output_filename('test-name.xx')
1018         self.assertTrue(os.path.exists(fname))
1019
1020     def testBlobFilename(self):
1021         """Test that generic blobs can be provided by filename"""
1022         data = self._DoReadFile('023_blob.dts')
1023         self.assertEqual(BLOB_DATA, data)
1024
1025     def testPackSorted(self):
1026         """Test that entries can be sorted"""
1027         self._SetupSplElf()
1028         data = self._DoReadFile('024_sorted.dts')
1029         self.assertEqual(tools.get_bytes(0, 1) + U_BOOT_SPL_DATA +
1030                          tools.get_bytes(0, 2) + U_BOOT_DATA, data)
1031
1032     def testPackZeroOffset(self):
1033         """Test that an entry at offset 0 is not given a new offset"""
1034         with self.assertRaises(ValueError) as e:
1035             self._DoTestFile('025_pack_zero_size.dts')
1036         self.assertIn("Node '/binman/u-boot-spl': Offset 0x0 (0) overlaps "
1037                       "with previous entry '/binman/u-boot' ending at 0x4 (4)",
1038                       str(e.exception))
1039
1040     def testPackUbootDtb(self):
1041         """Test that a device tree can be added to U-Boot"""
1042         data = self._DoReadFile('026_pack_u_boot_dtb.dts')
1043         self.assertEqual(U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA, data)
1044
1045     def testPackX86RomNoSize(self):
1046         """Test that the end-at-4gb property requires a size property"""
1047         with self.assertRaises(ValueError) as e:
1048             self._DoTestFile('027_pack_4gb_no_size.dts')
1049         self.assertIn("Image '/binman': Section size must be provided when "
1050                       "using end-at-4gb", str(e.exception))
1051
1052     def test4gbAndSkipAtStartTogether(self):
1053         """Test that the end-at-4gb and skip-at-size property can't be used
1054         together"""
1055         with self.assertRaises(ValueError) as e:
1056             self._DoTestFile('098_4gb_and_skip_at_start_together.dts')
1057         self.assertIn("Image '/binman': Provide either 'end-at-4gb' or "
1058                       "'skip-at-start'", str(e.exception))
1059
1060     def testPackX86RomOutside(self):
1061         """Test that the end-at-4gb property checks for offset boundaries"""
1062         with self.assertRaises(ValueError) as e:
1063             self._DoTestFile('028_pack_4gb_outside.dts')
1064         self.assertIn("Node '/binman/u-boot': Offset 0x0 (0) size 0x4 (4) "
1065                       "is outside the section '/binman' starting at "
1066                       '0xffffffe0 (4294967264) of size 0x20 (32)',
1067                       str(e.exception))
1068
1069     def testPackX86Rom(self):
1070         """Test that a basic x86 ROM can be created"""
1071         self._SetupSplElf()
1072         data = self._DoReadFile('029_x86_rom.dts')
1073         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 3) + U_BOOT_SPL_DATA +
1074                          tools.get_bytes(0, 2), data)
1075
1076     def testPackX86RomMeNoDesc(self):
1077         """Test that an invalid Intel descriptor entry is detected"""
1078         try:
1079             TestFunctional._MakeInputFile('descriptor-empty.bin', b'')
1080             with self.assertRaises(ValueError) as e:
1081                 self._DoTestFile('163_x86_rom_me_empty.dts')
1082             self.assertIn("Node '/binman/intel-descriptor': Cannot find Intel Flash Descriptor (FD) signature",
1083                           str(e.exception))
1084         finally:
1085             self._SetupDescriptor()
1086
1087     def testPackX86RomBadDesc(self):
1088         """Test that the Intel requires a descriptor entry"""
1089         with self.assertRaises(ValueError) as e:
1090             self._DoTestFile('030_x86_rom_me_no_desc.dts')
1091         self.assertIn("Node '/binman/intel-me': No offset set with "
1092                       "offset-unset: should another entry provide this correct "
1093                       "offset?", str(e.exception))
1094
1095     def testPackX86RomMe(self):
1096         """Test that an x86 ROM with an ME region can be created"""
1097         data = self._DoReadFile('031_x86_rom_me.dts')
1098         expected_desc = tools.read_file(self.TestFile('descriptor.bin'))
1099         if data[:0x1000] != expected_desc:
1100             self.fail('Expected descriptor binary at start of image')
1101         self.assertEqual(ME_DATA, data[0x1000:0x1000 + len(ME_DATA)])
1102
1103     def testPackVga(self):
1104         """Test that an image with a VGA binary can be created"""
1105         data = self._DoReadFile('032_intel_vga.dts')
1106         self.assertEqual(VGA_DATA, data[:len(VGA_DATA)])
1107
1108     def testPackStart16(self):
1109         """Test that an image with an x86 start16 region can be created"""
1110         data = self._DoReadFile('033_x86_start16.dts')
1111         self.assertEqual(X86_START16_DATA, data[:len(X86_START16_DATA)])
1112
1113     def testPackPowerpcMpc85xxBootpgResetvec(self):
1114         """Test that an image with powerpc-mpc85xx-bootpg-resetvec can be
1115         created"""
1116         data = self._DoReadFile('150_powerpc_mpc85xx_bootpg_resetvec.dts')
1117         self.assertEqual(PPC_MPC85XX_BR_DATA, data[:len(PPC_MPC85XX_BR_DATA)])
1118
1119     def _RunMicrocodeTest(self, dts_fname, nodtb_data, ucode_second=False):
1120         """Handle running a test for insertion of microcode
1121
1122         Args:
1123             dts_fname: Name of test .dts file
1124             nodtb_data: Data that we expect in the first section
1125             ucode_second: True if the microsecond entry is second instead of
1126                 third
1127
1128         Returns:
1129             Tuple:
1130                 Contents of first region (U-Boot or SPL)
1131                 Offset and size components of microcode pointer, as inserted
1132                     in the above (two 4-byte words)
1133         """
1134         data = self._DoReadFile(dts_fname, True)
1135
1136         # Now check the device tree has no microcode
1137         if ucode_second:
1138             ucode_content = data[len(nodtb_data):]
1139             ucode_pos = len(nodtb_data)
1140             dtb_with_ucode = ucode_content[16:]
1141             fdt_len = self.GetFdtLen(dtb_with_ucode)
1142         else:
1143             dtb_with_ucode = data[len(nodtb_data):]
1144             fdt_len = self.GetFdtLen(dtb_with_ucode)
1145             ucode_content = dtb_with_ucode[fdt_len:]
1146             ucode_pos = len(nodtb_data) + fdt_len
1147         fname = tools.get_output_filename('test.dtb')
1148         with open(fname, 'wb') as fd:
1149             fd.write(dtb_with_ucode)
1150         dtb = fdt.FdtScan(fname)
1151         ucode = dtb.GetNode('/microcode')
1152         self.assertTrue(ucode)
1153         for node in ucode.subnodes:
1154             self.assertFalse(node.props.get('data'))
1155
1156         # Check that the microcode appears immediately after the Fdt
1157         # This matches the concatenation of the data properties in
1158         # the /microcode/update@xxx nodes in 34_x86_ucode.dts.
1159         ucode_data = struct.pack('>4L', 0x12345678, 0x12345679, 0xabcd0000,
1160                                  0x78235609)
1161         self.assertEqual(ucode_data, ucode_content[:len(ucode_data)])
1162
1163         # Check that the microcode pointer was inserted. It should match the
1164         # expected offset and size
1165         pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos,
1166                                    len(ucode_data))
1167         u_boot = data[:len(nodtb_data)]
1168         return u_boot, pos_and_size
1169
1170     def testPackUbootMicrocode(self):
1171         """Test that x86 microcode can be handled correctly
1172
1173         We expect to see the following in the image, in order:
1174             u-boot-nodtb.bin with a microcode pointer inserted at the correct
1175                 place
1176             u-boot.dtb with the microcode removed
1177             the microcode
1178         """
1179         first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts',
1180                                                      U_BOOT_NODTB_DATA)
1181         self.assertEqual(b'nodtb with microcode' + pos_and_size +
1182                          b' somewhere in here', first)
1183
1184     def _RunPackUbootSingleMicrocode(self):
1185         """Test that x86 microcode can be handled correctly
1186
1187         We expect to see the following in the image, in order:
1188             u-boot-nodtb.bin with a microcode pointer inserted at the correct
1189                 place
1190             u-boot.dtb with the microcode
1191             an empty microcode region
1192         """
1193         # We need the libfdt library to run this test since only that allows
1194         # finding the offset of a property. This is required by
1195         # Entry_u_boot_dtb_with_ucode.ObtainContents().
1196         data = self._DoReadFile('035_x86_single_ucode.dts', True)
1197
1198         second = data[len(U_BOOT_NODTB_DATA):]
1199
1200         fdt_len = self.GetFdtLen(second)
1201         third = second[fdt_len:]
1202         second = second[:fdt_len]
1203
1204         ucode_data = struct.pack('>2L', 0x12345678, 0x12345679)
1205         self.assertIn(ucode_data, second)
1206         ucode_pos = second.find(ucode_data) + len(U_BOOT_NODTB_DATA)
1207
1208         # Check that the microcode pointer was inserted. It should match the
1209         # expected offset and size
1210         pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos,
1211                                    len(ucode_data))
1212         first = data[:len(U_BOOT_NODTB_DATA)]
1213         self.assertEqual(b'nodtb with microcode' + pos_and_size +
1214                          b' somewhere in here', first)
1215
1216     def testPackUbootSingleMicrocode(self):
1217         """Test that x86 microcode can be handled correctly with fdt_normal.
1218         """
1219         self._RunPackUbootSingleMicrocode()
1220
1221     def testUBootImg(self):
1222         """Test that u-boot.img can be put in a file"""
1223         data = self._DoReadFile('036_u_boot_img.dts')
1224         self.assertEqual(U_BOOT_IMG_DATA, data)
1225
1226     def testNoMicrocode(self):
1227         """Test that a missing microcode region is detected"""
1228         with self.assertRaises(ValueError) as e:
1229             self._DoReadFile('037_x86_no_ucode.dts', True)
1230         self.assertIn("Node '/binman/u-boot-dtb-with-ucode': No /microcode "
1231                       "node found in ", str(e.exception))
1232
1233     def testMicrocodeWithoutNode(self):
1234         """Test that a missing u-boot-dtb-with-ucode node is detected"""
1235         with self.assertRaises(ValueError) as e:
1236             self._DoReadFile('038_x86_ucode_missing_node.dts', True)
1237         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot find "
1238                 "microcode region u-boot-dtb-with-ucode", str(e.exception))
1239
1240     def testMicrocodeWithoutNode2(self):
1241         """Test that a missing u-boot-ucode node is detected"""
1242         with self.assertRaises(ValueError) as e:
1243             self._DoReadFile('039_x86_ucode_missing_node2.dts', True)
1244         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot find "
1245             "microcode region u-boot-ucode", str(e.exception))
1246
1247     def testMicrocodeWithoutPtrInElf(self):
1248         """Test that a U-Boot binary without the microcode symbol is detected"""
1249         # ELF file without a '_dt_ucode_base_size' symbol
1250         try:
1251             TestFunctional._MakeInputFile('u-boot',
1252                 tools.read_file(self.ElfTestFile('u_boot_no_ucode_ptr')))
1253
1254             with self.assertRaises(ValueError) as e:
1255                 self._RunPackUbootSingleMicrocode()
1256             self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot locate "
1257                     "_dt_ucode_base_size symbol in u-boot", str(e.exception))
1258
1259         finally:
1260             # Put the original file back
1261             TestFunctional._MakeInputFile('u-boot',
1262                 tools.read_file(self.ElfTestFile('u_boot_ucode_ptr')))
1263
1264     def testMicrocodeNotInImage(self):
1265         """Test that microcode must be placed within the image"""
1266         with self.assertRaises(ValueError) as e:
1267             self._DoReadFile('040_x86_ucode_not_in_image.dts', True)
1268         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Microcode "
1269                 "pointer _dt_ucode_base_size at fffffe14 is outside the "
1270                 "section ranging from 00000000 to 0000002e", str(e.exception))
1271
1272     def testWithoutMicrocode(self):
1273         """Test that we can cope with an image without microcode (e.g. qemu)"""
1274         TestFunctional._MakeInputFile('u-boot',
1275             tools.read_file(self.ElfTestFile('u_boot_no_ucode_ptr')))
1276         data, dtb, _, _ = self._DoReadFileDtb('044_x86_optional_ucode.dts', True)
1277
1278         # Now check the device tree has no microcode
1279         self.assertEqual(U_BOOT_NODTB_DATA, data[:len(U_BOOT_NODTB_DATA)])
1280         second = data[len(U_BOOT_NODTB_DATA):]
1281
1282         fdt_len = self.GetFdtLen(second)
1283         self.assertEqual(dtb, second[:fdt_len])
1284
1285         used_len = len(U_BOOT_NODTB_DATA) + fdt_len
1286         third = data[used_len:]
1287         self.assertEqual(tools.get_bytes(0, 0x200 - used_len), third)
1288
1289     def testUnknownPosSize(self):
1290         """Test that microcode must be placed within the image"""
1291         with self.assertRaises(ValueError) as e:
1292             self._DoReadFile('041_unknown_pos_size.dts', True)
1293         self.assertIn("Section '/binman': Unable to set offset/size for unknown "
1294                 "entry 'invalid-entry'", str(e.exception))
1295
1296     def testPackFsp(self):
1297         """Test that an image with a FSP binary can be created"""
1298         data = self._DoReadFile('042_intel_fsp.dts')
1299         self.assertEqual(FSP_DATA, data[:len(FSP_DATA)])
1300
1301     def testPackCmc(self):
1302         """Test that an image with a CMC binary can be created"""
1303         data = self._DoReadFile('043_intel_cmc.dts')
1304         self.assertEqual(CMC_DATA, data[:len(CMC_DATA)])
1305
1306     def testPackVbt(self):
1307         """Test that an image with a VBT binary can be created"""
1308         data = self._DoReadFile('046_intel_vbt.dts')
1309         self.assertEqual(VBT_DATA, data[:len(VBT_DATA)])
1310
1311     def testSplBssPad(self):
1312         """Test that we can pad SPL's BSS with zeros"""
1313         # ELF file with a '__bss_size' symbol
1314         self._SetupSplElf()
1315         data = self._DoReadFile('047_spl_bss_pad.dts')
1316         self.assertEqual(U_BOOT_SPL_DATA + tools.get_bytes(0, 10) + U_BOOT_DATA,
1317                          data)
1318
1319     def testSplBssPadMissing(self):
1320         """Test that a missing symbol is detected"""
1321         self._SetupSplElf('u_boot_ucode_ptr')
1322         with self.assertRaises(ValueError) as e:
1323             self._DoReadFile('047_spl_bss_pad.dts')
1324         self.assertIn('Expected __bss_size symbol in spl/u-boot-spl',
1325                       str(e.exception))
1326
1327     def testPackStart16Spl(self):
1328         """Test that an image with an x86 start16 SPL region can be created"""
1329         data = self._DoReadFile('048_x86_start16_spl.dts')
1330         self.assertEqual(X86_START16_SPL_DATA, data[:len(X86_START16_SPL_DATA)])
1331
1332     def _PackUbootSplMicrocode(self, dts, ucode_second=False):
1333         """Helper function for microcode tests
1334
1335         We expect to see the following in the image, in order:
1336             u-boot-spl-nodtb.bin with a microcode pointer inserted at the
1337                 correct place
1338             u-boot.dtb with the microcode removed
1339             the microcode
1340
1341         Args:
1342             dts: Device tree file to use for test
1343             ucode_second: True if the microsecond entry is second instead of
1344                 third
1345         """
1346         self._SetupSplElf('u_boot_ucode_ptr')
1347         first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA,
1348                                                      ucode_second=ucode_second)
1349         self.assertEqual(b'splnodtb with microc' + pos_and_size +
1350                          b'ter somewhere in here', first)
1351
1352     def testPackUbootSplMicrocode(self):
1353         """Test that x86 microcode can be handled correctly in SPL"""
1354         self._PackUbootSplMicrocode('049_x86_ucode_spl.dts')
1355
1356     def testPackUbootSplMicrocodeReorder(self):
1357         """Test that order doesn't matter for microcode entries
1358
1359         This is the same as testPackUbootSplMicrocode but when we process the
1360         u-boot-ucode entry we have not yet seen the u-boot-dtb-with-ucode
1361         entry, so we reply on binman to try later.
1362         """
1363         self._PackUbootSplMicrocode('058_x86_ucode_spl_needs_retry.dts',
1364                                     ucode_second=True)
1365
1366     def testPackMrc(self):
1367         """Test that an image with an MRC binary can be created"""
1368         data = self._DoReadFile('050_intel_mrc.dts')
1369         self.assertEqual(MRC_DATA, data[:len(MRC_DATA)])
1370
1371     def testSplDtb(self):
1372         """Test that an image with spl/u-boot-spl.dtb can be created"""
1373         data = self._DoReadFile('051_u_boot_spl_dtb.dts')
1374         self.assertEqual(U_BOOT_SPL_DTB_DATA, data[:len(U_BOOT_SPL_DTB_DATA)])
1375
1376     def testSplNoDtb(self):
1377         """Test that an image with spl/u-boot-spl-nodtb.bin can be created"""
1378         self._SetupSplElf()
1379         data = self._DoReadFile('052_u_boot_spl_nodtb.dts')
1380         self.assertEqual(U_BOOT_SPL_NODTB_DATA, data[:len(U_BOOT_SPL_NODTB_DATA)])
1381
1382     def checkSymbols(self, dts, base_data, u_boot_offset, entry_args=None,
1383                      use_expanded=False):
1384         """Check the image contains the expected symbol values
1385
1386         Args:
1387             dts: Device tree file to use for test
1388             base_data: Data before and after 'u-boot' section
1389             u_boot_offset: Offset of 'u-boot' section in image
1390             entry_args: Dict of entry args to supply to binman
1391                 key: arg name
1392                 value: value of that arg
1393             use_expanded: True to use expanded entries where available, e.g.
1394                 'u-boot-expanded' instead of 'u-boot'
1395         """
1396         elf_fname = self.ElfTestFile('u_boot_binman_syms')
1397         syms = elf.GetSymbols(elf_fname, ['binman', 'image'])
1398         addr = elf.GetSymbolAddress(elf_fname, '__image_copy_start')
1399         self.assertEqual(syms['_binman_u_boot_spl_any_prop_offset'].address,
1400                          addr)
1401
1402         self._SetupSplElf('u_boot_binman_syms')
1403         data = self._DoReadFileDtb(dts, entry_args=entry_args,
1404                                    use_expanded=use_expanded)[0]
1405         # The image should contain the symbols from u_boot_binman_syms.c
1406         # Note that image_pos is adjusted by the base address of the image,
1407         # which is 0x10 in our test image
1408         sym_values = struct.pack('<LQLL', 0x00,
1409                                  u_boot_offset + len(U_BOOT_DATA),
1410                                  0x10 + u_boot_offset, 0x04)
1411         expected = (sym_values + base_data[20:] +
1412                     tools.get_bytes(0xff, 1) + U_BOOT_DATA + sym_values +
1413                     base_data[20:])
1414         self.assertEqual(expected, data)
1415
1416     def testSymbols(self):
1417         """Test binman can assign symbols embedded in U-Boot"""
1418         self.checkSymbols('053_symbols.dts', U_BOOT_SPL_DATA, 0x18)
1419
1420     def testSymbolsNoDtb(self):
1421         """Test binman can assign symbols embedded in U-Boot SPL"""
1422         self.checkSymbols('196_symbols_nodtb.dts',
1423                           U_BOOT_SPL_NODTB_DATA + U_BOOT_SPL_DTB_DATA,
1424                           0x38)
1425
1426     def testPackUnitAddress(self):
1427         """Test that we support multiple binaries with the same name"""
1428         data = self._DoReadFile('054_unit_address.dts')
1429         self.assertEqual(U_BOOT_DATA + U_BOOT_DATA, data)
1430
1431     def testSections(self):
1432         """Basic test of sections"""
1433         data = self._DoReadFile('055_sections.dts')
1434         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
1435                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12) +
1436                     U_BOOT_DATA + tools.get_bytes(ord('&'), 4))
1437         self.assertEqual(expected, data)
1438
1439     def testMap(self):
1440         """Tests outputting a map of the images"""
1441         _, _, map_data, _ = self._DoReadFileDtb('055_sections.dts', map=True)
1442         self.assertEqual('''ImagePos    Offset      Size  Name
1443 00000000  00000000  00000028  main-section
1444 00000000   00000000  00000010  section@0
1445 00000000    00000000  00000004  u-boot
1446 00000010   00000010  00000010  section@1
1447 00000010    00000000  00000004  u-boot
1448 00000020   00000020  00000004  section@2
1449 00000020    00000000  00000004  u-boot
1450 ''', map_data)
1451
1452     def testNamePrefix(self):
1453         """Tests that name prefixes are used"""
1454         _, _, map_data, _ = self._DoReadFileDtb('056_name_prefix.dts', map=True)
1455         self.assertEqual('''ImagePos    Offset      Size  Name
1456 00000000  00000000  00000028  main-section
1457 00000000   00000000  00000010  section@0
1458 00000000    00000000  00000004  ro-u-boot
1459 00000010   00000010  00000010  section@1
1460 00000010    00000000  00000004  rw-u-boot
1461 ''', map_data)
1462
1463     def testUnknownContents(self):
1464         """Test that obtaining the contents works as expected"""
1465         with self.assertRaises(ValueError) as e:
1466             self._DoReadFile('057_unknown_contents.dts', True)
1467         self.assertIn("Image '/binman': Internal error: Could not complete "
1468                 "processing of contents: remaining ["
1469                 "<binman.etype._testing.Entry__testing ", str(e.exception))
1470
1471     def testBadChangeSize(self):
1472         """Test that trying to change the size of an entry fails"""
1473         try:
1474             state.SetAllowEntryExpansion(False)
1475             with self.assertRaises(ValueError) as e:
1476                 self._DoReadFile('059_change_size.dts', True)
1477             self.assertIn("Node '/binman/_testing': Cannot update entry size from 2 to 3",
1478                           str(e.exception))
1479         finally:
1480             state.SetAllowEntryExpansion(True)
1481
1482     def testUpdateFdt(self):
1483         """Test that we can update the device tree with offset/size info"""
1484         _, _, _, out_dtb_fname = self._DoReadFileDtb('060_fdt_update.dts',
1485                                                      update_dtb=True)
1486         dtb = fdt.Fdt(out_dtb_fname)
1487         dtb.Scan()
1488         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
1489         self.assertEqual({
1490             'image-pos': 0,
1491             'offset': 0,
1492             '_testing:offset': 32,
1493             '_testing:size': 2,
1494             '_testing:image-pos': 32,
1495             'section@0/u-boot:offset': 0,
1496             'section@0/u-boot:size': len(U_BOOT_DATA),
1497             'section@0/u-boot:image-pos': 0,
1498             'section@0:offset': 0,
1499             'section@0:size': 16,
1500             'section@0:image-pos': 0,
1501
1502             'section@1/u-boot:offset': 0,
1503             'section@1/u-boot:size': len(U_BOOT_DATA),
1504             'section@1/u-boot:image-pos': 16,
1505             'section@1:offset': 16,
1506             'section@1:size': 16,
1507             'section@1:image-pos': 16,
1508             'size': 40
1509         }, props)
1510
1511     def testUpdateFdtBad(self):
1512         """Test that we detect when ProcessFdt never completes"""
1513         with self.assertRaises(ValueError) as e:
1514             self._DoReadFileDtb('061_fdt_update_bad.dts', update_dtb=True)
1515         self.assertIn('Could not complete processing of Fdt: remaining '
1516                       '[<binman.etype._testing.Entry__testing',
1517                         str(e.exception))
1518
1519     def testEntryArgs(self):
1520         """Test passing arguments to entries from the command line"""
1521         entry_args = {
1522             'test-str-arg': 'test1',
1523             'test-int-arg': '456',
1524         }
1525         self._DoReadFileDtb('062_entry_args.dts', entry_args=entry_args)
1526         self.assertIn('image', control.images)
1527         entry = control.images['image'].GetEntries()['_testing']
1528         self.assertEqual('test0', entry.test_str_fdt)
1529         self.assertEqual('test1', entry.test_str_arg)
1530         self.assertEqual(123, entry.test_int_fdt)
1531         self.assertEqual(456, entry.test_int_arg)
1532
1533     def testEntryArgsMissing(self):
1534         """Test missing arguments and properties"""
1535         entry_args = {
1536             'test-int-arg': '456',
1537         }
1538         self._DoReadFileDtb('063_entry_args_missing.dts', entry_args=entry_args)
1539         entry = control.images['image'].GetEntries()['_testing']
1540         self.assertEqual('test0', entry.test_str_fdt)
1541         self.assertEqual(None, entry.test_str_arg)
1542         self.assertEqual(None, entry.test_int_fdt)
1543         self.assertEqual(456, entry.test_int_arg)
1544
1545     def testEntryArgsRequired(self):
1546         """Test missing arguments and properties"""
1547         entry_args = {
1548             'test-int-arg': '456',
1549         }
1550         with self.assertRaises(ValueError) as e:
1551             self._DoReadFileDtb('064_entry_args_required.dts')
1552         self.assertIn("Node '/binman/_testing': "
1553             'Missing required properties/entry args: test-str-arg, '
1554             'test-int-fdt, test-int-arg',
1555             str(e.exception))
1556
1557     def testEntryArgsInvalidFormat(self):
1558         """Test that an invalid entry-argument format is detected"""
1559         args = ['build', '-d', self.TestFile('064_entry_args_required.dts'),
1560                 '-ano-value']
1561         with self.assertRaises(ValueError) as e:
1562             self._DoBinman(*args)
1563         self.assertIn("Invalid entry arguemnt 'no-value'", str(e.exception))
1564
1565     def testEntryArgsInvalidInteger(self):
1566         """Test that an invalid entry-argument integer is detected"""
1567         entry_args = {
1568             'test-int-arg': 'abc',
1569         }
1570         with self.assertRaises(ValueError) as e:
1571             self._DoReadFileDtb('062_entry_args.dts', entry_args=entry_args)
1572         self.assertIn("Node '/binman/_testing': Cannot convert entry arg "
1573                       "'test-int-arg' (value 'abc') to integer",
1574             str(e.exception))
1575
1576     def testEntryArgsInvalidDatatype(self):
1577         """Test that an invalid entry-argument datatype is detected
1578
1579         This test could be written in entry_test.py except that it needs
1580         access to control.entry_args, which seems more than that module should
1581         be able to see.
1582         """
1583         entry_args = {
1584             'test-bad-datatype-arg': '12',
1585         }
1586         with self.assertRaises(ValueError) as e:
1587             self._DoReadFileDtb('065_entry_args_unknown_datatype.dts',
1588                                 entry_args=entry_args)
1589         self.assertIn('GetArg() internal error: Unknown data type ',
1590                       str(e.exception))
1591
1592     def testText(self):
1593         """Test for a text entry type"""
1594         entry_args = {
1595             'test-id': TEXT_DATA,
1596             'test-id2': TEXT_DATA2,
1597             'test-id3': TEXT_DATA3,
1598         }
1599         data, _, _, _ = self._DoReadFileDtb('066_text.dts',
1600                                             entry_args=entry_args)
1601         expected = (tools.to_bytes(TEXT_DATA) +
1602                     tools.get_bytes(0, 8 - len(TEXT_DATA)) +
1603                     tools.to_bytes(TEXT_DATA2) + tools.to_bytes(TEXT_DATA3) +
1604                     b'some text' + b'more text')
1605         self.assertEqual(expected, data)
1606
1607     def testEntryDocs(self):
1608         """Test for creation of entry documentation"""
1609         with test_util.capture_sys_output() as (stdout, stderr):
1610             control.WriteEntryDocs(control.GetEntryModules())
1611         self.assertTrue(len(stdout.getvalue()) > 0)
1612
1613     def testEntryDocsMissing(self):
1614         """Test handling of missing entry documentation"""
1615         with self.assertRaises(ValueError) as e:
1616             with test_util.capture_sys_output() as (stdout, stderr):
1617                 control.WriteEntryDocs(control.GetEntryModules(), 'u_boot')
1618         self.assertIn('Documentation is missing for modules: u_boot',
1619                       str(e.exception))
1620
1621     def testFmap(self):
1622         """Basic test of generation of a flashrom fmap"""
1623         data = self._DoReadFile('067_fmap.dts')
1624         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
1625         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
1626                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12))
1627         self.assertEqual(expected, data[:32])
1628         self.assertEqual(b'__FMAP__', fhdr.signature)
1629         self.assertEqual(1, fhdr.ver_major)
1630         self.assertEqual(0, fhdr.ver_minor)
1631         self.assertEqual(0, fhdr.base)
1632         expect_size = fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 5
1633         self.assertEqual(16 + 16 + expect_size, fhdr.image_size)
1634         self.assertEqual(b'FMAP', fhdr.name)
1635         self.assertEqual(5, fhdr.nareas)
1636         fiter = iter(fentries)
1637
1638         fentry = next(fiter)
1639         self.assertEqual(b'SECTION0', fentry.name)
1640         self.assertEqual(0, fentry.offset)
1641         self.assertEqual(16, fentry.size)
1642         self.assertEqual(0, fentry.flags)
1643
1644         fentry = next(fiter)
1645         self.assertEqual(b'RO_U_BOOT', fentry.name)
1646         self.assertEqual(0, fentry.offset)
1647         self.assertEqual(4, fentry.size)
1648         self.assertEqual(0, fentry.flags)
1649
1650         fentry = next(fiter)
1651         self.assertEqual(b'SECTION1', fentry.name)
1652         self.assertEqual(16, fentry.offset)
1653         self.assertEqual(16, fentry.size)
1654         self.assertEqual(0, fentry.flags)
1655
1656         fentry = next(fiter)
1657         self.assertEqual(b'RW_U_BOOT', fentry.name)
1658         self.assertEqual(16, fentry.offset)
1659         self.assertEqual(4, fentry.size)
1660         self.assertEqual(0, fentry.flags)
1661
1662         fentry = next(fiter)
1663         self.assertEqual(b'FMAP', fentry.name)
1664         self.assertEqual(32, fentry.offset)
1665         self.assertEqual(expect_size, fentry.size)
1666         self.assertEqual(0, fentry.flags)
1667
1668     def testBlobNamedByArg(self):
1669         """Test we can add a blob with the filename coming from an entry arg"""
1670         entry_args = {
1671             'cros-ec-rw-path': 'ecrw.bin',
1672         }
1673         self._DoReadFileDtb('068_blob_named_by_arg.dts', entry_args=entry_args)
1674
1675     def testFill(self):
1676         """Test for an fill entry type"""
1677         data = self._DoReadFile('069_fill.dts')
1678         expected = tools.get_bytes(0xff, 8) + tools.get_bytes(0, 8)
1679         self.assertEqual(expected, data)
1680
1681     def testFillNoSize(self):
1682         """Test for an fill entry type with no size"""
1683         with self.assertRaises(ValueError) as e:
1684             self._DoReadFile('070_fill_no_size.dts')
1685         self.assertIn("'fill' entry must have a size property",
1686                       str(e.exception))
1687
1688     def _HandleGbbCommand(self, pipe_list):
1689         """Fake calls to the futility utility"""
1690         if pipe_list[0][0] == 'futility':
1691             fname = pipe_list[0][-1]
1692             # Append our GBB data to the file, which will happen every time the
1693             # futility command is called.
1694             with open(fname, 'ab') as fd:
1695                 fd.write(GBB_DATA)
1696             return command.CommandResult()
1697
1698     def testGbb(self):
1699         """Test for the Chromium OS Google Binary Block"""
1700         command.test_result = self._HandleGbbCommand
1701         entry_args = {
1702             'keydir': 'devkeys',
1703             'bmpblk': 'bmpblk.bin',
1704         }
1705         data, _, _, _ = self._DoReadFileDtb('071_gbb.dts', entry_args=entry_args)
1706
1707         # Since futility
1708         expected = (GBB_DATA + GBB_DATA + tools.get_bytes(0, 8) +
1709                     tools.get_bytes(0, 0x2180 - 16))
1710         self.assertEqual(expected, data)
1711
1712     def testGbbTooSmall(self):
1713         """Test for the Chromium OS Google Binary Block being large enough"""
1714         with self.assertRaises(ValueError) as e:
1715             self._DoReadFileDtb('072_gbb_too_small.dts')
1716         self.assertIn("Node '/binman/gbb': GBB is too small",
1717                       str(e.exception))
1718
1719     def testGbbNoSize(self):
1720         """Test for the Chromium OS Google Binary Block having a size"""
1721         with self.assertRaises(ValueError) as e:
1722             self._DoReadFileDtb('073_gbb_no_size.dts')
1723         self.assertIn("Node '/binman/gbb': GBB must have a fixed size",
1724                       str(e.exception))
1725
1726     def testGbbMissing(self):
1727         """Test that binman still produces an image if futility is missing"""
1728         entry_args = {
1729             'keydir': 'devkeys',
1730         }
1731         with test_util.capture_sys_output() as (_, stderr):
1732             self._DoTestFile('071_gbb.dts', force_missing_bintools='futility',
1733                              entry_args=entry_args)
1734         err = stderr.getvalue()
1735         self.assertRegex(err,
1736                          "Image 'main-section'.*missing bintools.*: futility")
1737
1738     def _HandleVblockCommand(self, pipe_list):
1739         """Fake calls to the futility utility
1740
1741         The expected pipe is:
1742
1743            [('futility', 'vbutil_firmware', '--vblock',
1744              'vblock.vblock', '--keyblock', 'devkeys/firmware.keyblock',
1745              '--signprivate', 'devkeys/firmware_data_key.vbprivk',
1746              '--version', '1', '--fv', 'input.vblock', '--kernelkey',
1747              'devkeys/kernel_subkey.vbpubk', '--flags', '1')]
1748
1749         This writes to the output file (here, 'vblock.vblock'). If
1750         self._hash_data is False, it writes VBLOCK_DATA, else it writes a hash
1751         of the input data (here, 'input.vblock').
1752         """
1753         if pipe_list[0][0] == 'futility':
1754             fname = pipe_list[0][3]
1755             with open(fname, 'wb') as fd:
1756                 if self._hash_data:
1757                     infile = pipe_list[0][11]
1758                     m = hashlib.sha256()
1759                     data = tools.read_file(infile)
1760                     m.update(data)
1761                     fd.write(m.digest())
1762                 else:
1763                     fd.write(VBLOCK_DATA)
1764
1765             return command.CommandResult()
1766
1767     def testVblock(self):
1768         """Test for the Chromium OS Verified Boot Block"""
1769         self._hash_data = False
1770         command.test_result = self._HandleVblockCommand
1771         entry_args = {
1772             'keydir': 'devkeys',
1773         }
1774         data, _, _, _ = self._DoReadFileDtb('074_vblock.dts',
1775                                             entry_args=entry_args)
1776         expected = U_BOOT_DATA + VBLOCK_DATA + U_BOOT_DTB_DATA
1777         self.assertEqual(expected, data)
1778
1779     def testVblockNoContent(self):
1780         """Test we detect a vblock which has no content to sign"""
1781         with self.assertRaises(ValueError) as e:
1782             self._DoReadFile('075_vblock_no_content.dts')
1783         self.assertIn("Node '/binman/vblock': Collection must have a 'content' "
1784                       'property', str(e.exception))
1785
1786     def testVblockBadPhandle(self):
1787         """Test that we detect a vblock with an invalid phandle in contents"""
1788         with self.assertRaises(ValueError) as e:
1789             self._DoReadFile('076_vblock_bad_phandle.dts')
1790         self.assertIn("Node '/binman/vblock': Cannot find node for phandle "
1791                       '1000', str(e.exception))
1792
1793     def testVblockBadEntry(self):
1794         """Test that we detect an entry that points to a non-entry"""
1795         with self.assertRaises(ValueError) as e:
1796             self._DoReadFile('077_vblock_bad_entry.dts')
1797         self.assertIn("Node '/binman/vblock': Cannot find entry for node "
1798                       "'other'", str(e.exception))
1799
1800     def testVblockContent(self):
1801         """Test that the vblock signs the right data"""
1802         self._hash_data = True
1803         command.test_result = self._HandleVblockCommand
1804         entry_args = {
1805             'keydir': 'devkeys',
1806         }
1807         data = self._DoReadFileDtb(
1808             '189_vblock_content.dts', use_real_dtb=True, update_dtb=True,
1809             entry_args=entry_args)[0]
1810         hashlen = 32  # SHA256 hash is 32 bytes
1811         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
1812         hashval = data[-hashlen:]
1813         dtb = data[len(U_BOOT_DATA):-hashlen]
1814
1815         expected_data = U_BOOT_DATA + dtb
1816
1817         # The hashval should be a hash of the dtb
1818         m = hashlib.sha256()
1819         m.update(expected_data)
1820         expected_hashval = m.digest()
1821         self.assertEqual(expected_hashval, hashval)
1822
1823     def testVblockMissing(self):
1824         """Test that binman still produces an image if futility is missing"""
1825         entry_args = {
1826             'keydir': 'devkeys',
1827         }
1828         with test_util.capture_sys_output() as (_, stderr):
1829             self._DoTestFile('074_vblock.dts',
1830                              force_missing_bintools='futility',
1831                              entry_args=entry_args)
1832         err = stderr.getvalue()
1833         self.assertRegex(err,
1834                          "Image 'main-section'.*missing bintools.*: futility")
1835
1836     def testTpl(self):
1837         """Test that an image with TPL and its device tree can be created"""
1838         # ELF file with a '__bss_size' symbol
1839         self._SetupTplElf()
1840         data = self._DoReadFile('078_u_boot_tpl.dts')
1841         self.assertEqual(U_BOOT_TPL_DATA + U_BOOT_TPL_DTB_DATA, data)
1842
1843     def testUsesPos(self):
1844         """Test that the 'pos' property cannot be used anymore"""
1845         with self.assertRaises(ValueError) as e:
1846            data = self._DoReadFile('079_uses_pos.dts')
1847         self.assertIn("Node '/binman/u-boot': Please use 'offset' instead of "
1848                       "'pos'", str(e.exception))
1849
1850     def testFillZero(self):
1851         """Test for an fill entry type with a size of 0"""
1852         data = self._DoReadFile('080_fill_empty.dts')
1853         self.assertEqual(tools.get_bytes(0, 16), data)
1854
1855     def testTextMissing(self):
1856         """Test for a text entry type where there is no text"""
1857         with self.assertRaises(ValueError) as e:
1858             self._DoReadFileDtb('066_text.dts',)
1859         self.assertIn("Node '/binman/text': No value provided for text label "
1860                       "'test-id'", str(e.exception))
1861
1862     def testPackStart16Tpl(self):
1863         """Test that an image with an x86 start16 TPL region can be created"""
1864         data = self._DoReadFile('081_x86_start16_tpl.dts')
1865         self.assertEqual(X86_START16_TPL_DATA, data[:len(X86_START16_TPL_DATA)])
1866
1867     def testSelectImage(self):
1868         """Test that we can select which images to build"""
1869         expected = 'Skipping images: image1'
1870
1871         # We should only get the expected message in verbose mode
1872         for verbosity in (0, 2):
1873             with test_util.capture_sys_output() as (stdout, stderr):
1874                 retcode = self._DoTestFile('006_dual_image.dts',
1875                                            verbosity=verbosity,
1876                                            images=['image2'])
1877             self.assertEqual(0, retcode)
1878             if verbosity:
1879                 self.assertIn(expected, stdout.getvalue())
1880             else:
1881                 self.assertNotIn(expected, stdout.getvalue())
1882
1883             self.assertFalse(os.path.exists(tools.get_output_filename('image1.bin')))
1884             self.assertTrue(os.path.exists(tools.get_output_filename('image2.bin')))
1885             self._CleanupOutputDir()
1886
1887     def testUpdateFdtAll(self):
1888         """Test that all device trees are updated with offset/size info"""
1889         data = self._DoReadFileRealDtb('082_fdt_update_all.dts')
1890
1891         base_expected = {
1892             'section:image-pos': 0,
1893             'u-boot-tpl-dtb:size': 513,
1894             'u-boot-spl-dtb:size': 513,
1895             'u-boot-spl-dtb:offset': 493,
1896             'image-pos': 0,
1897             'section/u-boot-dtb:image-pos': 0,
1898             'u-boot-spl-dtb:image-pos': 493,
1899             'section/u-boot-dtb:size': 493,
1900             'u-boot-tpl-dtb:image-pos': 1006,
1901             'section/u-boot-dtb:offset': 0,
1902             'section:size': 493,
1903             'offset': 0,
1904             'section:offset': 0,
1905             'u-boot-tpl-dtb:offset': 1006,
1906             'size': 1519
1907         }
1908
1909         # We expect three device-tree files in the output, one after the other.
1910         # Read them in sequence. We look for an 'spl' property in the SPL tree,
1911         # and 'tpl' in the TPL tree, to make sure they are distinct from the
1912         # main U-Boot tree. All three should have the same postions and offset.
1913         start = 0
1914         for item in ['', 'spl', 'tpl']:
1915             dtb = fdt.Fdt.FromData(data[start:])
1916             dtb.Scan()
1917             props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS +
1918                                       ['spl', 'tpl'])
1919             expected = dict(base_expected)
1920             if item:
1921                 expected[item] = 0
1922             self.assertEqual(expected, props)
1923             start += dtb._fdt_obj.totalsize()
1924
1925     def testUpdateFdtOutput(self):
1926         """Test that output DTB files are updated"""
1927         try:
1928             data, dtb_data, _, _ = self._DoReadFileDtb('082_fdt_update_all.dts',
1929                     use_real_dtb=True, update_dtb=True, reset_dtbs=False)
1930
1931             # Unfortunately, compiling a source file always results in a file
1932             # called source.dtb (see fdt_util.EnsureCompiled()). The test
1933             # source file (e.g. test/075_fdt_update_all.dts) thus does not enter
1934             # binman as a file called u-boot.dtb. To fix this, copy the file
1935             # over to the expected place.
1936             start = 0
1937             for fname in ['u-boot.dtb.out', 'spl/u-boot-spl.dtb.out',
1938                           'tpl/u-boot-tpl.dtb.out']:
1939                 dtb = fdt.Fdt.FromData(data[start:])
1940                 size = dtb._fdt_obj.totalsize()
1941                 pathname = tools.get_output_filename(os.path.split(fname)[1])
1942                 outdata = tools.read_file(pathname)
1943                 name = os.path.split(fname)[0]
1944
1945                 if name:
1946                     orig_indata = self._GetDtbContentsForSplTpl(dtb_data, name)
1947                 else:
1948                     orig_indata = dtb_data
1949                 self.assertNotEqual(outdata, orig_indata,
1950                         "Expected output file '%s' be updated" % pathname)
1951                 self.assertEqual(outdata, data[start:start + size],
1952                         "Expected output file '%s' to match output image" %
1953                         pathname)
1954                 start += size
1955         finally:
1956             self._ResetDtbs()
1957
1958     def _decompress(self, data):
1959         return comp_util.decompress(data, 'lz4')
1960
1961     def testCompress(self):
1962         """Test compression of blobs"""
1963         self._CheckLz4()
1964         data, _, _, out_dtb_fname = self._DoReadFileDtb('083_compress.dts',
1965                                             use_real_dtb=True, update_dtb=True)
1966         dtb = fdt.Fdt(out_dtb_fname)
1967         dtb.Scan()
1968         props = self._GetPropTree(dtb, ['size', 'uncomp-size'])
1969         orig = self._decompress(data)
1970         self.assertEquals(COMPRESS_DATA, orig)
1971
1972         # Do a sanity check on various fields
1973         image = control.images['image']
1974         entries = image.GetEntries()
1975         self.assertEqual(1, len(entries))
1976
1977         entry = entries['blob']
1978         self.assertEqual(COMPRESS_DATA, entry.uncomp_data)
1979         self.assertEqual(len(COMPRESS_DATA), entry.uncomp_size)
1980         orig = self._decompress(entry.data)
1981         self.assertEqual(orig, entry.uncomp_data)
1982
1983         self.assertEqual(image.data, entry.data)
1984
1985         expected = {
1986             'blob:uncomp-size': len(COMPRESS_DATA),
1987             'blob:size': len(data),
1988             'size': len(data),
1989             }
1990         self.assertEqual(expected, props)
1991
1992     def testFiles(self):
1993         """Test bringing in multiple files"""
1994         data = self._DoReadFile('084_files.dts')
1995         self.assertEqual(FILES_DATA, data)
1996
1997     def testFilesCompress(self):
1998         """Test bringing in multiple files and compressing them"""
1999         self._CheckLz4()
2000         data = self._DoReadFile('085_files_compress.dts')
2001
2002         image = control.images['image']
2003         entries = image.GetEntries()
2004         files = entries['files']
2005         entries = files._entries
2006
2007         orig = b''
2008         for i in range(1, 3):
2009             key = '%d.dat' % i
2010             start = entries[key].image_pos
2011             len = entries[key].size
2012             chunk = data[start:start + len]
2013             orig += self._decompress(chunk)
2014
2015         self.assertEqual(FILES_DATA, orig)
2016
2017     def testFilesMissing(self):
2018         """Test missing files"""
2019         with self.assertRaises(ValueError) as e:
2020             data = self._DoReadFile('086_files_none.dts')
2021         self.assertIn("Node '/binman/files': Pattern \'files/*.none\' matched "
2022                       'no files', str(e.exception))
2023
2024     def testFilesNoPattern(self):
2025         """Test missing files"""
2026         with self.assertRaises(ValueError) as e:
2027             data = self._DoReadFile('087_files_no_pattern.dts')
2028         self.assertIn("Node '/binman/files': Missing 'pattern' property",
2029                       str(e.exception))
2030
2031     def testExtendSize(self):
2032         """Test an extending entry"""
2033         data, _, map_data, _ = self._DoReadFileDtb('088_extend_size.dts',
2034                                                    map=True)
2035         expect = (tools.get_bytes(ord('a'), 8) + U_BOOT_DATA +
2036                   MRC_DATA + tools.get_bytes(ord('b'), 1) + U_BOOT_DATA +
2037                   tools.get_bytes(ord('c'), 8) + U_BOOT_DATA +
2038                   tools.get_bytes(ord('d'), 8))
2039         self.assertEqual(expect, data)
2040         self.assertEqual('''ImagePos    Offset      Size  Name
2041 00000000  00000000  00000028  main-section
2042 00000000   00000000  00000008  fill
2043 00000008   00000008  00000004  u-boot
2044 0000000c   0000000c  00000004  section
2045 0000000c    00000000  00000003  intel-mrc
2046 00000010   00000010  00000004  u-boot2
2047 00000014   00000014  0000000c  section2
2048 00000014    00000000  00000008  fill
2049 0000001c    00000008  00000004  u-boot
2050 00000020   00000020  00000008  fill2
2051 ''', map_data)
2052
2053     def testExtendSizeBad(self):
2054         """Test an extending entry which fails to provide contents"""
2055         with test_util.capture_sys_output() as (stdout, stderr):
2056             with self.assertRaises(ValueError) as e:
2057                 self._DoReadFileDtb('089_extend_size_bad.dts', map=True)
2058         self.assertIn("Node '/binman/_testing': Cannot obtain contents when "
2059                       'expanding entry', str(e.exception))
2060
2061     def testHash(self):
2062         """Test hashing of the contents of an entry"""
2063         _, _, _, out_dtb_fname = self._DoReadFileDtb('090_hash.dts',
2064                 use_real_dtb=True, update_dtb=True)
2065         dtb = fdt.Fdt(out_dtb_fname)
2066         dtb.Scan()
2067         hash_node = dtb.GetNode('/binman/u-boot/hash').props['value']
2068         m = hashlib.sha256()
2069         m.update(U_BOOT_DATA)
2070         self.assertEqual(m.digest(), b''.join(hash_node.value))
2071
2072     def testHashNoAlgo(self):
2073         with self.assertRaises(ValueError) as e:
2074             self._DoReadFileDtb('091_hash_no_algo.dts', update_dtb=True)
2075         self.assertIn("Node \'/binman/u-boot\': Missing \'algo\' property for "
2076                       'hash node', str(e.exception))
2077
2078     def testHashBadAlgo(self):
2079         with self.assertRaises(ValueError) as e:
2080             self._DoReadFileDtb('092_hash_bad_algo.dts', update_dtb=True)
2081         self.assertIn("Node '/binman/u-boot': Unknown hash algorithm 'invalid'",
2082                       str(e.exception))
2083
2084     def testHashSection(self):
2085         """Test hashing of the contents of an entry"""
2086         _, _, _, out_dtb_fname = self._DoReadFileDtb('099_hash_section.dts',
2087                 use_real_dtb=True, update_dtb=True)
2088         dtb = fdt.Fdt(out_dtb_fname)
2089         dtb.Scan()
2090         hash_node = dtb.GetNode('/binman/section/hash').props['value']
2091         m = hashlib.sha256()
2092         m.update(U_BOOT_DATA)
2093         m.update(tools.get_bytes(ord('a'), 16))
2094         self.assertEqual(m.digest(), b''.join(hash_node.value))
2095
2096     def testPackUBootTplMicrocode(self):
2097         """Test that x86 microcode can be handled correctly in TPL
2098
2099         We expect to see the following in the image, in order:
2100             u-boot-tpl-nodtb.bin with a microcode pointer inserted at the correct
2101                 place
2102             u-boot-tpl.dtb with the microcode removed
2103             the microcode
2104         """
2105         self._SetupTplElf('u_boot_ucode_ptr')
2106         first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts',
2107                                                      U_BOOT_TPL_NODTB_DATA)
2108         self.assertEqual(b'tplnodtb with microc' + pos_and_size +
2109                          b'ter somewhere in here', first)
2110
2111     def testFmapX86(self):
2112         """Basic test of generation of a flashrom fmap"""
2113         data = self._DoReadFile('094_fmap_x86.dts')
2114         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
2115         expected = U_BOOT_DATA + MRC_DATA + tools.get_bytes(ord('a'), 32 - 7)
2116         self.assertEqual(expected, data[:32])
2117         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
2118
2119         self.assertEqual(0x100, fhdr.image_size)
2120
2121         self.assertEqual(0, fentries[0].offset)
2122         self.assertEqual(4, fentries[0].size)
2123         self.assertEqual(b'U_BOOT', fentries[0].name)
2124
2125         self.assertEqual(4, fentries[1].offset)
2126         self.assertEqual(3, fentries[1].size)
2127         self.assertEqual(b'INTEL_MRC', fentries[1].name)
2128
2129         self.assertEqual(32, fentries[2].offset)
2130         self.assertEqual(fmap_util.FMAP_HEADER_LEN +
2131                          fmap_util.FMAP_AREA_LEN * 3, fentries[2].size)
2132         self.assertEqual(b'FMAP', fentries[2].name)
2133
2134     def testFmapX86Section(self):
2135         """Basic test of generation of a flashrom fmap"""
2136         data = self._DoReadFile('095_fmap_x86_section.dts')
2137         expected = U_BOOT_DATA + MRC_DATA + tools.get_bytes(ord('b'), 32 - 7)
2138         self.assertEqual(expected, data[:32])
2139         fhdr, fentries = fmap_util.DecodeFmap(data[36:])
2140
2141         self.assertEqual(0x180, fhdr.image_size)
2142         expect_size = fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 4
2143         fiter = iter(fentries)
2144
2145         fentry = next(fiter)
2146         self.assertEqual(b'U_BOOT', fentry.name)
2147         self.assertEqual(0, fentry.offset)
2148         self.assertEqual(4, fentry.size)
2149
2150         fentry = next(fiter)
2151         self.assertEqual(b'SECTION', fentry.name)
2152         self.assertEqual(4, fentry.offset)
2153         self.assertEqual(0x20 + expect_size, fentry.size)
2154
2155         fentry = next(fiter)
2156         self.assertEqual(b'INTEL_MRC', fentry.name)
2157         self.assertEqual(4, fentry.offset)
2158         self.assertEqual(3, fentry.size)
2159
2160         fentry = next(fiter)
2161         self.assertEqual(b'FMAP', fentry.name)
2162         self.assertEqual(36, fentry.offset)
2163         self.assertEqual(expect_size, fentry.size)
2164
2165     def testElf(self):
2166         """Basic test of ELF entries"""
2167         self._SetupSplElf()
2168         self._SetupTplElf()
2169         with open(self.ElfTestFile('bss_data'), 'rb') as fd:
2170             TestFunctional._MakeInputFile('-boot', fd.read())
2171         data = self._DoReadFile('096_elf.dts')
2172
2173     def testElfStrip(self):
2174         """Basic test of ELF entries"""
2175         self._SetupSplElf()
2176         with open(self.ElfTestFile('bss_data'), 'rb') as fd:
2177             TestFunctional._MakeInputFile('-boot', fd.read())
2178         data = self._DoReadFile('097_elf_strip.dts')
2179
2180     def testPackOverlapMap(self):
2181         """Test that overlapping regions are detected"""
2182         with test_util.capture_sys_output() as (stdout, stderr):
2183             with self.assertRaises(ValueError) as e:
2184                 self._DoTestFile('014_pack_overlap.dts', map=True)
2185         map_fname = tools.get_output_filename('image.map')
2186         self.assertEqual("Wrote map file '%s' to show errors\n" % map_fname,
2187                          stdout.getvalue())
2188
2189         # We should not get an inmage, but there should be a map file
2190         self.assertFalse(os.path.exists(tools.get_output_filename('image.bin')))
2191         self.assertTrue(os.path.exists(map_fname))
2192         map_data = tools.read_file(map_fname, binary=False)
2193         self.assertEqual('''ImagePos    Offset      Size  Name
2194 <none>    00000000  00000008  main-section
2195 <none>     00000000  00000004  u-boot
2196 <none>     00000003  00000004  u-boot-align
2197 ''', map_data)
2198
2199     def testPackRefCode(self):
2200         """Test that an image with an Intel Reference code binary works"""
2201         data = self._DoReadFile('100_intel_refcode.dts')
2202         self.assertEqual(REFCODE_DATA, data[:len(REFCODE_DATA)])
2203
2204     def testSectionOffset(self):
2205         """Tests use of a section with an offset"""
2206         data, _, map_data, _ = self._DoReadFileDtb('101_sections_offset.dts',
2207                                                    map=True)
2208         self.assertEqual('''ImagePos    Offset      Size  Name
2209 00000000  00000000  00000038  main-section
2210 00000004   00000004  00000010  section@0
2211 00000004    00000000  00000004  u-boot
2212 00000018   00000018  00000010  section@1
2213 00000018    00000000  00000004  u-boot
2214 0000002c   0000002c  00000004  section@2
2215 0000002c    00000000  00000004  u-boot
2216 ''', map_data)
2217         self.assertEqual(data,
2218                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2219                              tools.get_bytes(0x21, 12) +
2220                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2221                              tools.get_bytes(0x61, 12) +
2222                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2223                              tools.get_bytes(0x26, 8))
2224
2225     def testCbfsRaw(self):
2226         """Test base handling of a Coreboot Filesystem (CBFS)
2227
2228         The exact contents of the CBFS is verified by similar tests in
2229         cbfs_util_test.py. The tests here merely check that the files added to
2230         the CBFS can be found in the final image.
2231         """
2232         data = self._DoReadFile('102_cbfs_raw.dts')
2233         size = 0xb0
2234
2235         cbfs = cbfs_util.CbfsReader(data)
2236         self.assertEqual(size, cbfs.rom_size)
2237
2238         self.assertIn('u-boot-dtb', cbfs.files)
2239         cfile = cbfs.files['u-boot-dtb']
2240         self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
2241
2242     def testCbfsArch(self):
2243         """Test on non-x86 architecture"""
2244         data = self._DoReadFile('103_cbfs_raw_ppc.dts')
2245         size = 0x100
2246
2247         cbfs = cbfs_util.CbfsReader(data)
2248         self.assertEqual(size, cbfs.rom_size)
2249
2250         self.assertIn('u-boot-dtb', cbfs.files)
2251         cfile = cbfs.files['u-boot-dtb']
2252         self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
2253
2254     def testCbfsStage(self):
2255         """Tests handling of a Coreboot Filesystem (CBFS)"""
2256         if not elf.ELF_TOOLS:
2257             self.skipTest('Python elftools not available')
2258         elf_fname = os.path.join(self._indir, 'cbfs-stage.elf')
2259         elf.MakeElf(elf_fname, U_BOOT_DATA, U_BOOT_DTB_DATA)
2260         size = 0xb0
2261
2262         data = self._DoReadFile('104_cbfs_stage.dts')
2263         cbfs = cbfs_util.CbfsReader(data)
2264         self.assertEqual(size, cbfs.rom_size)
2265
2266         self.assertIn('u-boot', cbfs.files)
2267         cfile = cbfs.files['u-boot']
2268         self.assertEqual(U_BOOT_DATA + U_BOOT_DTB_DATA, cfile.data)
2269
2270     def testCbfsRawCompress(self):
2271         """Test handling of compressing raw files"""
2272         self._CheckLz4()
2273         data = self._DoReadFile('105_cbfs_raw_compress.dts')
2274         size = 0x140
2275
2276         cbfs = cbfs_util.CbfsReader(data)
2277         self.assertIn('u-boot', cbfs.files)
2278         cfile = cbfs.files['u-boot']
2279         self.assertEqual(COMPRESS_DATA, cfile.data)
2280
2281     def testCbfsBadArch(self):
2282         """Test handling of a bad architecture"""
2283         with self.assertRaises(ValueError) as e:
2284             self._DoReadFile('106_cbfs_bad_arch.dts')
2285         self.assertIn("Invalid architecture 'bad-arch'", str(e.exception))
2286
2287     def testCbfsNoSize(self):
2288         """Test handling of a missing size property"""
2289         with self.assertRaises(ValueError) as e:
2290             self._DoReadFile('107_cbfs_no_size.dts')
2291         self.assertIn('entry must have a size property', str(e.exception))
2292
2293     def testCbfsNoContents(self):
2294         """Test handling of a CBFS entry which does not provide contentsy"""
2295         with self.assertRaises(ValueError) as e:
2296             self._DoReadFile('108_cbfs_no_contents.dts')
2297         self.assertIn('Could not complete processing of contents',
2298                       str(e.exception))
2299
2300     def testCbfsBadCompress(self):
2301         """Test handling of a bad architecture"""
2302         with self.assertRaises(ValueError) as e:
2303             self._DoReadFile('109_cbfs_bad_compress.dts')
2304         self.assertIn("Invalid compression in 'u-boot': 'invalid-algo'",
2305                       str(e.exception))
2306
2307     def testCbfsNamedEntries(self):
2308         """Test handling of named entries"""
2309         data = self._DoReadFile('110_cbfs_name.dts')
2310
2311         cbfs = cbfs_util.CbfsReader(data)
2312         self.assertIn('FRED', cbfs.files)
2313         cfile1 = cbfs.files['FRED']
2314         self.assertEqual(U_BOOT_DATA, cfile1.data)
2315
2316         self.assertIn('hello', cbfs.files)
2317         cfile2 = cbfs.files['hello']
2318         self.assertEqual(U_BOOT_DTB_DATA, cfile2.data)
2319
2320     def _SetupIfwi(self, fname):
2321         """Set up to run an IFWI test
2322
2323         Args:
2324             fname: Filename of input file to provide (fitimage.bin or ifwi.bin)
2325         """
2326         self._SetupSplElf()
2327         self._SetupTplElf()
2328
2329         # Intel Integrated Firmware Image (IFWI) file
2330         with gzip.open(self.TestFile('%s.gz' % fname), 'rb') as fd:
2331             data = fd.read()
2332         TestFunctional._MakeInputFile(fname,data)
2333
2334     def _CheckIfwi(self, data):
2335         """Check that an image with an IFWI contains the correct output
2336
2337         Args:
2338             data: Conents of output file
2339         """
2340         expected_desc = tools.read_file(self.TestFile('descriptor.bin'))
2341         if data[:0x1000] != expected_desc:
2342             self.fail('Expected descriptor binary at start of image')
2343
2344         # We expect to find the TPL wil in subpart IBBP entry IBBL
2345         image_fname = tools.get_output_filename('image.bin')
2346         tpl_fname = tools.get_output_filename('tpl.out')
2347         ifwitool = bintool.Bintool.create('ifwitool')
2348         ifwitool.extract(image_fname, 'IBBP', 'IBBL', tpl_fname)
2349
2350         tpl_data = tools.read_file(tpl_fname)
2351         self.assertEqual(U_BOOT_TPL_DATA, tpl_data[:len(U_BOOT_TPL_DATA)])
2352
2353     def testPackX86RomIfwi(self):
2354         """Test that an x86 ROM with Integrated Firmware Image can be created"""
2355         self._SetupIfwi('fitimage.bin')
2356         data = self._DoReadFile('111_x86_rom_ifwi.dts')
2357         self._CheckIfwi(data)
2358
2359     def testPackX86RomIfwiNoDesc(self):
2360         """Test that an x86 ROM with IFWI can be created from an ifwi.bin file"""
2361         self._SetupIfwi('ifwi.bin')
2362         data = self._DoReadFile('112_x86_rom_ifwi_nodesc.dts')
2363         self._CheckIfwi(data)
2364
2365     def testPackX86RomIfwiNoData(self):
2366         """Test that an x86 ROM with IFWI handles missing data"""
2367         self._SetupIfwi('ifwi.bin')
2368         with self.assertRaises(ValueError) as e:
2369             data = self._DoReadFile('113_x86_rom_ifwi_nodata.dts')
2370         self.assertIn('Could not complete processing of contents',
2371                       str(e.exception))
2372
2373     def testIfwiMissing(self):
2374         """Test that binman still produces an image if ifwitool is missing"""
2375         self._SetupIfwi('fitimage.bin')
2376         with test_util.capture_sys_output() as (_, stderr):
2377             self._DoTestFile('111_x86_rom_ifwi.dts',
2378                              force_missing_bintools='ifwitool')
2379         err = stderr.getvalue()
2380         self.assertRegex(err,
2381                          "Image 'main-section'.*missing bintools.*: ifwitool")
2382
2383     def testCbfsOffset(self):
2384         """Test a CBFS with files at particular offsets
2385
2386         Like all CFBS tests, this is just checking the logic that calls
2387         cbfs_util. See cbfs_util_test for fully tests (e.g. test_cbfs_offset()).
2388         """
2389         data = self._DoReadFile('114_cbfs_offset.dts')
2390         size = 0x200
2391
2392         cbfs = cbfs_util.CbfsReader(data)
2393         self.assertEqual(size, cbfs.rom_size)
2394
2395         self.assertIn('u-boot', cbfs.files)
2396         cfile = cbfs.files['u-boot']
2397         self.assertEqual(U_BOOT_DATA, cfile.data)
2398         self.assertEqual(0x40, cfile.cbfs_offset)
2399
2400         self.assertIn('u-boot-dtb', cbfs.files)
2401         cfile2 = cbfs.files['u-boot-dtb']
2402         self.assertEqual(U_BOOT_DTB_DATA, cfile2.data)
2403         self.assertEqual(0x140, cfile2.cbfs_offset)
2404
2405     def testFdtmap(self):
2406         """Test an FDT map can be inserted in the image"""
2407         data = self.data = self._DoReadFileRealDtb('115_fdtmap.dts')
2408         fdtmap_data = data[len(U_BOOT_DATA):]
2409         magic = fdtmap_data[:8]
2410         self.assertEqual(b'_FDTMAP_', magic)
2411         self.assertEqual(tools.get_bytes(0, 8), fdtmap_data[8:16])
2412
2413         fdt_data = fdtmap_data[16:]
2414         dtb = fdt.Fdt.FromData(fdt_data)
2415         dtb.Scan()
2416         props = self._GetPropTree(dtb, BASE_DTB_PROPS, prefix='/')
2417         self.assertEqual({
2418             'image-pos': 0,
2419             'offset': 0,
2420             'u-boot:offset': 0,
2421             'u-boot:size': len(U_BOOT_DATA),
2422             'u-boot:image-pos': 0,
2423             'fdtmap:image-pos': 4,
2424             'fdtmap:offset': 4,
2425             'fdtmap:size': len(fdtmap_data),
2426             'size': len(data),
2427         }, props)
2428
2429     def testFdtmapNoMatch(self):
2430         """Check handling of an FDT map when the section cannot be found"""
2431         self.data = self._DoReadFileRealDtb('115_fdtmap.dts')
2432
2433         # Mangle the section name, which should cause a mismatch between the
2434         # correct FDT path and the one expected by the section
2435         image = control.images['image']
2436         image._node.path += '-suffix'
2437         entries = image.GetEntries()
2438         fdtmap = entries['fdtmap']
2439         with self.assertRaises(ValueError) as e:
2440             fdtmap._GetFdtmap()
2441         self.assertIn("Cannot locate node for path '/binman-suffix'",
2442                       str(e.exception))
2443
2444     def testFdtmapHeader(self):
2445         """Test an FDT map and image header can be inserted in the image"""
2446         data = self.data = self._DoReadFileRealDtb('116_fdtmap_hdr.dts')
2447         fdtmap_pos = len(U_BOOT_DATA)
2448         fdtmap_data = data[fdtmap_pos:]
2449         fdt_data = fdtmap_data[16:]
2450         dtb = fdt.Fdt.FromData(fdt_data)
2451         fdt_size = dtb.GetFdtObj().totalsize()
2452         hdr_data = data[-8:]
2453         self.assertEqual(b'BinM', hdr_data[:4])
2454         offset = struct.unpack('<I', hdr_data[4:])[0] & 0xffffffff
2455         self.assertEqual(fdtmap_pos - 0x400, offset - (1 << 32))
2456
2457     def testFdtmapHeaderStart(self):
2458         """Test an image header can be inserted at the image start"""
2459         data = self.data = self._DoReadFileRealDtb('117_fdtmap_hdr_start.dts')
2460         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
2461         hdr_data = data[:8]
2462         self.assertEqual(b'BinM', hdr_data[:4])
2463         offset = struct.unpack('<I', hdr_data[4:])[0]
2464         self.assertEqual(fdtmap_pos, offset)
2465
2466     def testFdtmapHeaderPos(self):
2467         """Test an image header can be inserted at a chosen position"""
2468         data = self.data = self._DoReadFileRealDtb('118_fdtmap_hdr_pos.dts')
2469         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
2470         hdr_data = data[0x80:0x88]
2471         self.assertEqual(b'BinM', hdr_data[:4])
2472         offset = struct.unpack('<I', hdr_data[4:])[0]
2473         self.assertEqual(fdtmap_pos, offset)
2474
2475     def testHeaderMissingFdtmap(self):
2476         """Test an image header requires an fdtmap"""
2477         with self.assertRaises(ValueError) as e:
2478             self.data = self._DoReadFileRealDtb('119_fdtmap_hdr_missing.dts')
2479         self.assertIn("'image_header' section must have an 'fdtmap' sibling",
2480                       str(e.exception))
2481
2482     def testHeaderNoLocation(self):
2483         """Test an image header with a no specified location is detected"""
2484         with self.assertRaises(ValueError) as e:
2485             self.data = self._DoReadFileRealDtb('120_hdr_no_location.dts')
2486         self.assertIn("Invalid location 'None', expected 'start' or 'end'",
2487                       str(e.exception))
2488
2489     def testEntryExpand(self):
2490         """Test extending an entry after it is packed"""
2491         data = self._DoReadFile('121_entry_extend.dts')
2492         self.assertEqual(b'aaa', data[:3])
2493         self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)])
2494         self.assertEqual(b'aaa', data[-3:])
2495
2496     def testEntryExtendBad(self):
2497         """Test extending an entry after it is packed, twice"""
2498         with self.assertRaises(ValueError) as e:
2499             self._DoReadFile('122_entry_extend_twice.dts')
2500         self.assertIn("Image '/binman': Entries changed size after packing",
2501                       str(e.exception))
2502
2503     def testEntryExtendSection(self):
2504         """Test extending an entry within a section after it is packed"""
2505         data = self._DoReadFile('123_entry_extend_section.dts')
2506         self.assertEqual(b'aaa', data[:3])
2507         self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)])
2508         self.assertEqual(b'aaa', data[-3:])
2509
2510     def testCompressDtb(self):
2511         """Test that compress of device-tree files is supported"""
2512         self._CheckLz4()
2513         data = self.data = self._DoReadFileRealDtb('124_compress_dtb.dts')
2514         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
2515         comp_data = data[len(U_BOOT_DATA):]
2516         orig = self._decompress(comp_data)
2517         dtb = fdt.Fdt.FromData(orig)
2518         dtb.Scan()
2519         props = self._GetPropTree(dtb, ['size', 'uncomp-size'])
2520         expected = {
2521             'u-boot:size': len(U_BOOT_DATA),
2522             'u-boot-dtb:uncomp-size': len(orig),
2523             'u-boot-dtb:size': len(comp_data),
2524             'size': len(data),
2525             }
2526         self.assertEqual(expected, props)
2527
2528     def testCbfsUpdateFdt(self):
2529         """Test that we can update the device tree with CBFS offset/size info"""
2530         self._CheckLz4()
2531         data, _, _, out_dtb_fname = self._DoReadFileDtb('125_cbfs_update.dts',
2532                                                         update_dtb=True)
2533         dtb = fdt.Fdt(out_dtb_fname)
2534         dtb.Scan()
2535         props = self._GetPropTree(dtb, BASE_DTB_PROPS + ['uncomp-size'])
2536         del props['cbfs/u-boot:size']
2537         self.assertEqual({
2538             'offset': 0,
2539             'size': len(data),
2540             'image-pos': 0,
2541             'cbfs:offset': 0,
2542             'cbfs:size': len(data),
2543             'cbfs:image-pos': 0,
2544             'cbfs/u-boot:offset': 0x38,
2545             'cbfs/u-boot:uncomp-size': len(U_BOOT_DATA),
2546             'cbfs/u-boot:image-pos': 0x38,
2547             'cbfs/u-boot-dtb:offset': 0xb8,
2548             'cbfs/u-boot-dtb:size': len(U_BOOT_DATA),
2549             'cbfs/u-boot-dtb:image-pos': 0xb8,
2550             }, props)
2551
2552     def testCbfsBadType(self):
2553         """Test an image header with a no specified location is detected"""
2554         with self.assertRaises(ValueError) as e:
2555             self._DoReadFile('126_cbfs_bad_type.dts')
2556         self.assertIn("Unknown cbfs-type 'badtype'", str(e.exception))
2557
2558     def testList(self):
2559         """Test listing the files in an image"""
2560         self._CheckLz4()
2561         data = self._DoReadFile('127_list.dts')
2562         image = control.images['image']
2563         entries = image.BuildEntryList()
2564         self.assertEqual(7, len(entries))
2565
2566         ent = entries[0]
2567         self.assertEqual(0, ent.indent)
2568         self.assertEqual('main-section', ent.name)
2569         self.assertEqual('section', ent.etype)
2570         self.assertEqual(len(data), ent.size)
2571         self.assertEqual(0, ent.image_pos)
2572         self.assertEqual(None, ent.uncomp_size)
2573         self.assertEqual(0, ent.offset)
2574
2575         ent = entries[1]
2576         self.assertEqual(1, ent.indent)
2577         self.assertEqual('u-boot', ent.name)
2578         self.assertEqual('u-boot', ent.etype)
2579         self.assertEqual(len(U_BOOT_DATA), ent.size)
2580         self.assertEqual(0, ent.image_pos)
2581         self.assertEqual(None, ent.uncomp_size)
2582         self.assertEqual(0, ent.offset)
2583
2584         ent = entries[2]
2585         self.assertEqual(1, ent.indent)
2586         self.assertEqual('section', ent.name)
2587         self.assertEqual('section', ent.etype)
2588         section_size = ent.size
2589         self.assertEqual(0x100, ent.image_pos)
2590         self.assertEqual(None, ent.uncomp_size)
2591         self.assertEqual(0x100, ent.offset)
2592
2593         ent = entries[3]
2594         self.assertEqual(2, ent.indent)
2595         self.assertEqual('cbfs', ent.name)
2596         self.assertEqual('cbfs', ent.etype)
2597         self.assertEqual(0x400, ent.size)
2598         self.assertEqual(0x100, ent.image_pos)
2599         self.assertEqual(None, ent.uncomp_size)
2600         self.assertEqual(0, ent.offset)
2601
2602         ent = entries[4]
2603         self.assertEqual(3, ent.indent)
2604         self.assertEqual('u-boot', ent.name)
2605         self.assertEqual('u-boot', ent.etype)
2606         self.assertEqual(len(U_BOOT_DATA), ent.size)
2607         self.assertEqual(0x138, ent.image_pos)
2608         self.assertEqual(None, ent.uncomp_size)
2609         self.assertEqual(0x38, ent.offset)
2610
2611         ent = entries[5]
2612         self.assertEqual(3, ent.indent)
2613         self.assertEqual('u-boot-dtb', ent.name)
2614         self.assertEqual('text', ent.etype)
2615         self.assertGreater(len(COMPRESS_DATA), ent.size)
2616         self.assertEqual(0x178, ent.image_pos)
2617         self.assertEqual(len(COMPRESS_DATA), ent.uncomp_size)
2618         self.assertEqual(0x78, ent.offset)
2619
2620         ent = entries[6]
2621         self.assertEqual(2, ent.indent)
2622         self.assertEqual('u-boot-dtb', ent.name)
2623         self.assertEqual('u-boot-dtb', ent.etype)
2624         self.assertEqual(0x500, ent.image_pos)
2625         self.assertEqual(len(U_BOOT_DTB_DATA), ent.uncomp_size)
2626         dtb_size = ent.size
2627         # Compressing this data expands it since headers are added
2628         self.assertGreater(dtb_size, len(U_BOOT_DTB_DATA))
2629         self.assertEqual(0x400, ent.offset)
2630
2631         self.assertEqual(len(data), 0x100 + section_size)
2632         self.assertEqual(section_size, 0x400 + dtb_size)
2633
2634     def testFindFdtmap(self):
2635         """Test locating an FDT map in an image"""
2636         self._CheckLz4()
2637         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2638         image = control.images['image']
2639         entries = image.GetEntries()
2640         entry = entries['fdtmap']
2641         self.assertEqual(entry.image_pos, fdtmap.LocateFdtmap(data))
2642
2643     def testFindFdtmapMissing(self):
2644         """Test failing to locate an FDP map"""
2645         data = self._DoReadFile('005_simple.dts')
2646         self.assertEqual(None, fdtmap.LocateFdtmap(data))
2647
2648     def testFindImageHeader(self):
2649         """Test locating a image header"""
2650         self._CheckLz4()
2651         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2652         image = control.images['image']
2653         entries = image.GetEntries()
2654         entry = entries['fdtmap']
2655         # The header should point to the FDT map
2656         self.assertEqual(entry.image_pos, image_header.LocateHeaderOffset(data))
2657
2658     def testFindImageHeaderStart(self):
2659         """Test locating a image header located at the start of an image"""
2660         data = self.data = self._DoReadFileRealDtb('117_fdtmap_hdr_start.dts')
2661         image = control.images['image']
2662         entries = image.GetEntries()
2663         entry = entries['fdtmap']
2664         # The header should point to the FDT map
2665         self.assertEqual(entry.image_pos, image_header.LocateHeaderOffset(data))
2666
2667     def testFindImageHeaderMissing(self):
2668         """Test failing to locate an image header"""
2669         data = self._DoReadFile('005_simple.dts')
2670         self.assertEqual(None, image_header.LocateHeaderOffset(data))
2671
2672     def testReadImage(self):
2673         """Test reading an image and accessing its FDT map"""
2674         self._CheckLz4()
2675         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2676         image_fname = tools.get_output_filename('image.bin')
2677         orig_image = control.images['image']
2678         image = Image.FromFile(image_fname)
2679         self.assertEqual(orig_image.GetEntries().keys(),
2680                          image.GetEntries().keys())
2681
2682         orig_entry = orig_image.GetEntries()['fdtmap']
2683         entry = image.GetEntries()['fdtmap']
2684         self.assertEquals(orig_entry.offset, entry.offset)
2685         self.assertEquals(orig_entry.size, entry.size)
2686         self.assertEquals(orig_entry.image_pos, entry.image_pos)
2687
2688     def testReadImageNoHeader(self):
2689         """Test accessing an image's FDT map without an image header"""
2690         self._CheckLz4()
2691         data = self._DoReadFileRealDtb('129_decode_image_nohdr.dts')
2692         image_fname = tools.get_output_filename('image.bin')
2693         image = Image.FromFile(image_fname)
2694         self.assertTrue(isinstance(image, Image))
2695         self.assertEqual('image', image.image_name[-5:])
2696
2697     def testReadImageFail(self):
2698         """Test failing to read an image image's FDT map"""
2699         self._DoReadFile('005_simple.dts')
2700         image_fname = tools.get_output_filename('image.bin')
2701         with self.assertRaises(ValueError) as e:
2702             image = Image.FromFile(image_fname)
2703         self.assertIn("Cannot find FDT map in image", str(e.exception))
2704
2705     def testListCmd(self):
2706         """Test listing the files in an image using an Fdtmap"""
2707         self._CheckLz4()
2708         data = self._DoReadFileRealDtb('130_list_fdtmap.dts')
2709
2710         # lz4 compression size differs depending on the version
2711         image = control.images['image']
2712         entries = image.GetEntries()
2713         section_size = entries['section'].size
2714         fdt_size = entries['section'].GetEntries()['u-boot-dtb'].size
2715         fdtmap_offset = entries['fdtmap'].offset
2716
2717         try:
2718             tmpdir, updated_fname = self._SetupImageInTmpdir()
2719             with test_util.capture_sys_output() as (stdout, stderr):
2720                 self._DoBinman('ls', '-i', updated_fname)
2721         finally:
2722             shutil.rmtree(tmpdir)
2723         lines = stdout.getvalue().splitlines()
2724         expected = [
2725 'Name              Image-pos  Size  Entry-type    Offset  Uncomp-size',
2726 '----------------------------------------------------------------------',
2727 'main-section              0   c00  section            0',
2728 '  u-boot                  0     4  u-boot             0',
2729 '  section               100   %x  section          100' % section_size,
2730 '    cbfs                100   400  cbfs               0',
2731 '      u-boot            138     4  u-boot            38',
2732 '      u-boot-dtb        180   105  u-boot-dtb        80          3c9',
2733 '    u-boot-dtb          500   %x  u-boot-dtb       400          3c9' % fdt_size,
2734 '  fdtmap                %x   3bd  fdtmap           %x' %
2735         (fdtmap_offset, fdtmap_offset),
2736 '  image-header          bf8     8  image-header     bf8',
2737             ]
2738         self.assertEqual(expected, lines)
2739
2740     def testListCmdFail(self):
2741         """Test failing to list an image"""
2742         self._DoReadFile('005_simple.dts')
2743         try:
2744             tmpdir, updated_fname = self._SetupImageInTmpdir()
2745             with self.assertRaises(ValueError) as e:
2746                 self._DoBinman('ls', '-i', updated_fname)
2747         finally:
2748             shutil.rmtree(tmpdir)
2749         self.assertIn("Cannot find FDT map in image", str(e.exception))
2750
2751     def _RunListCmd(self, paths, expected):
2752         """List out entries and check the result
2753
2754         Args:
2755             paths: List of paths to pass to the list command
2756             expected: Expected list of filenames to be returned, in order
2757         """
2758         self._CheckLz4()
2759         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2760         image_fname = tools.get_output_filename('image.bin')
2761         image = Image.FromFile(image_fname)
2762         lines = image.GetListEntries(paths)[1]
2763         files = [line[0].strip() for line in lines[1:]]
2764         self.assertEqual(expected, files)
2765
2766     def testListCmdSection(self):
2767         """Test listing the files in a section"""
2768         self._RunListCmd(['section'],
2769             ['section', 'cbfs', 'u-boot', 'u-boot-dtb', 'u-boot-dtb'])
2770
2771     def testListCmdFile(self):
2772         """Test listing a particular file"""
2773         self._RunListCmd(['*u-boot-dtb'], ['u-boot-dtb', 'u-boot-dtb'])
2774
2775     def testListCmdWildcard(self):
2776         """Test listing a wildcarded file"""
2777         self._RunListCmd(['*boot*'],
2778             ['u-boot', 'u-boot', 'u-boot-dtb', 'u-boot-dtb'])
2779
2780     def testListCmdWildcardMulti(self):
2781         """Test listing a wildcarded file"""
2782         self._RunListCmd(['*cb*', '*head*'],
2783             ['cbfs', 'u-boot', 'u-boot-dtb', 'image-header'])
2784
2785     def testListCmdEmpty(self):
2786         """Test listing a wildcarded file"""
2787         self._RunListCmd(['nothing'], [])
2788
2789     def testListCmdPath(self):
2790         """Test listing the files in a sub-entry of a section"""
2791         self._RunListCmd(['section/cbfs'], ['cbfs', 'u-boot', 'u-boot-dtb'])
2792
2793     def _RunExtractCmd(self, entry_name, decomp=True):
2794         """Extract an entry from an image
2795
2796         Args:
2797             entry_name: Entry name to extract
2798             decomp: True to decompress the data if compressed, False to leave
2799                 it in its raw uncompressed format
2800
2801         Returns:
2802             data from entry
2803         """
2804         self._CheckLz4()
2805         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2806         image_fname = tools.get_output_filename('image.bin')
2807         return control.ReadEntry(image_fname, entry_name, decomp)
2808
2809     def testExtractSimple(self):
2810         """Test extracting a single file"""
2811         data = self._RunExtractCmd('u-boot')
2812         self.assertEqual(U_BOOT_DATA, data)
2813
2814     def testExtractSection(self):
2815         """Test extracting the files in a section"""
2816         data = self._RunExtractCmd('section')
2817         cbfs_data = data[:0x400]
2818         cbfs = cbfs_util.CbfsReader(cbfs_data)
2819         self.assertEqual(['u-boot', 'u-boot-dtb', ''], list(cbfs.files.keys()))
2820         dtb_data = data[0x400:]
2821         dtb = self._decompress(dtb_data)
2822         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2823
2824     def testExtractCompressed(self):
2825         """Test extracting compressed data"""
2826         data = self._RunExtractCmd('section/u-boot-dtb')
2827         self.assertEqual(EXTRACT_DTB_SIZE, len(data))
2828
2829     def testExtractRaw(self):
2830         """Test extracting compressed data without decompressing it"""
2831         data = self._RunExtractCmd('section/u-boot-dtb', decomp=False)
2832         dtb = self._decompress(data)
2833         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2834
2835     def testExtractCbfs(self):
2836         """Test extracting CBFS data"""
2837         data = self._RunExtractCmd('section/cbfs/u-boot')
2838         self.assertEqual(U_BOOT_DATA, data)
2839
2840     def testExtractCbfsCompressed(self):
2841         """Test extracting CBFS compressed data"""
2842         data = self._RunExtractCmd('section/cbfs/u-boot-dtb')
2843         self.assertEqual(EXTRACT_DTB_SIZE, len(data))
2844
2845     def testExtractCbfsRaw(self):
2846         """Test extracting CBFS compressed data without decompressing it"""
2847         data = self._RunExtractCmd('section/cbfs/u-boot-dtb', decomp=False)
2848         dtb = comp_util.decompress(data, 'lzma', with_header=False)
2849         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2850
2851     def testExtractBadEntry(self):
2852         """Test extracting a bad section path"""
2853         with self.assertRaises(ValueError) as e:
2854             self._RunExtractCmd('section/does-not-exist')
2855         self.assertIn("Entry 'does-not-exist' not found in '/section'",
2856                       str(e.exception))
2857
2858     def testExtractMissingFile(self):
2859         """Test extracting file that does not exist"""
2860         with self.assertRaises(IOError) as e:
2861             control.ReadEntry('missing-file', 'name')
2862
2863     def testExtractBadFile(self):
2864         """Test extracting an invalid file"""
2865         fname = os.path.join(self._indir, 'badfile')
2866         tools.write_file(fname, b'')
2867         with self.assertRaises(ValueError) as e:
2868             control.ReadEntry(fname, 'name')
2869
2870     def testExtractCmd(self):
2871         """Test extracting a file fron an image on the command line"""
2872         self._CheckLz4()
2873         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2874         fname = os.path.join(self._indir, 'output.extact')
2875         try:
2876             tmpdir, updated_fname = self._SetupImageInTmpdir()
2877             with test_util.capture_sys_output() as (stdout, stderr):
2878                 self._DoBinman('extract', '-i', updated_fname, 'u-boot',
2879                                '-f', fname)
2880         finally:
2881             shutil.rmtree(tmpdir)
2882         data = tools.read_file(fname)
2883         self.assertEqual(U_BOOT_DATA, data)
2884
2885     def testExtractOneEntry(self):
2886         """Test extracting a single entry fron an image """
2887         self._CheckLz4()
2888         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2889         image_fname = tools.get_output_filename('image.bin')
2890         fname = os.path.join(self._indir, 'output.extact')
2891         control.ExtractEntries(image_fname, fname, None, ['u-boot'])
2892         data = tools.read_file(fname)
2893         self.assertEqual(U_BOOT_DATA, data)
2894
2895     def _CheckExtractOutput(self, decomp):
2896         """Helper to test file output with and without decompression
2897
2898         Args:
2899             decomp: True to decompress entry data, False to output it raw
2900         """
2901         def _CheckPresent(entry_path, expect_data, expect_size=None):
2902             """Check and remove expected file
2903
2904             This checks the data/size of a file and removes the file both from
2905             the outfiles set and from the output directory. Once all files are
2906             processed, both the set and directory should be empty.
2907
2908             Args:
2909                 entry_path: Entry path
2910                 expect_data: Data to expect in file, or None to skip check
2911                 expect_size: Size of data to expect in file, or None to skip
2912             """
2913             path = os.path.join(outdir, entry_path)
2914             data = tools.read_file(path)
2915             os.remove(path)
2916             if expect_data:
2917                 self.assertEqual(expect_data, data)
2918             elif expect_size:
2919                 self.assertEqual(expect_size, len(data))
2920             outfiles.remove(path)
2921
2922         def _CheckDirPresent(name):
2923             """Remove expected directory
2924
2925             This gives an error if the directory does not exist as expected
2926
2927             Args:
2928                 name: Name of directory to remove
2929             """
2930             path = os.path.join(outdir, name)
2931             os.rmdir(path)
2932
2933         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2934         image_fname = tools.get_output_filename('image.bin')
2935         outdir = os.path.join(self._indir, 'extract')
2936         einfos = control.ExtractEntries(image_fname, None, outdir, [], decomp)
2937
2938         # Create a set of all file that were output (should be 9)
2939         outfiles = set()
2940         for root, dirs, files in os.walk(outdir):
2941             outfiles |= set([os.path.join(root, fname) for fname in files])
2942         self.assertEqual(9, len(outfiles))
2943         self.assertEqual(9, len(einfos))
2944
2945         image = control.images['image']
2946         entries = image.GetEntries()
2947
2948         # Check the 9 files in various ways
2949         section = entries['section']
2950         section_entries = section.GetEntries()
2951         cbfs_entries = section_entries['cbfs'].GetEntries()
2952         _CheckPresent('u-boot', U_BOOT_DATA)
2953         _CheckPresent('section/cbfs/u-boot', U_BOOT_DATA)
2954         dtb_len = EXTRACT_DTB_SIZE
2955         if not decomp:
2956             dtb_len = cbfs_entries['u-boot-dtb'].size
2957         _CheckPresent('section/cbfs/u-boot-dtb', None, dtb_len)
2958         if not decomp:
2959             dtb_len = section_entries['u-boot-dtb'].size
2960         _CheckPresent('section/u-boot-dtb', None, dtb_len)
2961
2962         fdtmap = entries['fdtmap']
2963         _CheckPresent('fdtmap', fdtmap.data)
2964         hdr = entries['image-header']
2965         _CheckPresent('image-header', hdr.data)
2966
2967         _CheckPresent('section/root', section.data)
2968         cbfs = section_entries['cbfs']
2969         _CheckPresent('section/cbfs/root', cbfs.data)
2970         data = tools.read_file(image_fname)
2971         _CheckPresent('root', data)
2972
2973         # There should be no files left. Remove all the directories to check.
2974         # If there are any files/dirs remaining, one of these checks will fail.
2975         self.assertEqual(0, len(outfiles))
2976         _CheckDirPresent('section/cbfs')
2977         _CheckDirPresent('section')
2978         _CheckDirPresent('')
2979         self.assertFalse(os.path.exists(outdir))
2980
2981     def testExtractAllEntries(self):
2982         """Test extracting all entries"""
2983         self._CheckLz4()
2984         self._CheckExtractOutput(decomp=True)
2985
2986     def testExtractAllEntriesRaw(self):
2987         """Test extracting all entries without decompressing them"""
2988         self._CheckLz4()
2989         self._CheckExtractOutput(decomp=False)
2990
2991     def testExtractSelectedEntries(self):
2992         """Test extracting some entries"""
2993         self._CheckLz4()
2994         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2995         image_fname = tools.get_output_filename('image.bin')
2996         outdir = os.path.join(self._indir, 'extract')
2997         einfos = control.ExtractEntries(image_fname, None, outdir,
2998                                         ['*cb*', '*head*'])
2999
3000         # File output is tested by testExtractAllEntries(), so just check that
3001         # the expected entries are selected
3002         names = [einfo.name for einfo in einfos]
3003         self.assertEqual(names,
3004                          ['cbfs', 'u-boot', 'u-boot-dtb', 'image-header'])
3005
3006     def testExtractNoEntryPaths(self):
3007         """Test extracting some entries"""
3008         self._CheckLz4()
3009         self._DoReadFileRealDtb('130_list_fdtmap.dts')
3010         image_fname = tools.get_output_filename('image.bin')
3011         with self.assertRaises(ValueError) as e:
3012             control.ExtractEntries(image_fname, 'fname', None, [])
3013         self.assertIn('Must specify an entry path to write with -f',
3014                       str(e.exception))
3015
3016     def testExtractTooManyEntryPaths(self):
3017         """Test extracting some entries"""
3018         self._CheckLz4()
3019         self._DoReadFileRealDtb('130_list_fdtmap.dts')
3020         image_fname = tools.get_output_filename('image.bin')
3021         with self.assertRaises(ValueError) as e:
3022             control.ExtractEntries(image_fname, 'fname', None, ['a', 'b'])
3023         self.assertIn('Must specify exactly one entry path to write with -f',
3024                       str(e.exception))
3025
3026     def testPackAlignSection(self):
3027         """Test that sections can have alignment"""
3028         self._DoReadFile('131_pack_align_section.dts')
3029
3030         self.assertIn('image', control.images)
3031         image = control.images['image']
3032         entries = image.GetEntries()
3033         self.assertEqual(3, len(entries))
3034
3035         # First u-boot
3036         self.assertIn('u-boot', entries)
3037         entry = entries['u-boot']
3038         self.assertEqual(0, entry.offset)
3039         self.assertEqual(0, entry.image_pos)
3040         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3041         self.assertEqual(len(U_BOOT_DATA), entry.size)
3042
3043         # Section0
3044         self.assertIn('section0', entries)
3045         section0 = entries['section0']
3046         self.assertEqual(0x10, section0.offset)
3047         self.assertEqual(0x10, section0.image_pos)
3048         self.assertEqual(len(U_BOOT_DATA), section0.size)
3049
3050         # Second u-boot
3051         section_entries = section0.GetEntries()
3052         self.assertIn('u-boot', section_entries)
3053         entry = section_entries['u-boot']
3054         self.assertEqual(0, entry.offset)
3055         self.assertEqual(0x10, entry.image_pos)
3056         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3057         self.assertEqual(len(U_BOOT_DATA), entry.size)
3058
3059         # Section1
3060         self.assertIn('section1', entries)
3061         section1 = entries['section1']
3062         self.assertEqual(0x14, section1.offset)
3063         self.assertEqual(0x14, section1.image_pos)
3064         self.assertEqual(0x20, section1.size)
3065
3066         # Second u-boot
3067         section_entries = section1.GetEntries()
3068         self.assertIn('u-boot', section_entries)
3069         entry = section_entries['u-boot']
3070         self.assertEqual(0, entry.offset)
3071         self.assertEqual(0x14, entry.image_pos)
3072         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3073         self.assertEqual(len(U_BOOT_DATA), entry.size)
3074
3075         # Section2
3076         self.assertIn('section2', section_entries)
3077         section2 = section_entries['section2']
3078         self.assertEqual(0x4, section2.offset)
3079         self.assertEqual(0x18, section2.image_pos)
3080         self.assertEqual(4, section2.size)
3081
3082         # Third u-boot
3083         section_entries = section2.GetEntries()
3084         self.assertIn('u-boot', section_entries)
3085         entry = section_entries['u-boot']
3086         self.assertEqual(0, entry.offset)
3087         self.assertEqual(0x18, entry.image_pos)
3088         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3089         self.assertEqual(len(U_BOOT_DATA), entry.size)
3090
3091     def _RunReplaceCmd(self, entry_name, data, decomp=True, allow_resize=True,
3092                        dts='132_replace.dts'):
3093         """Replace an entry in an image
3094
3095         This writes the entry data to update it, then opens the updated file and
3096         returns the value that it now finds there.
3097
3098         Args:
3099             entry_name: Entry name to replace
3100             data: Data to replace it with
3101             decomp: True to compress the data if needed, False if data is
3102                 already compressed so should be used as is
3103             allow_resize: True to allow entries to change size, False to raise
3104                 an exception
3105
3106         Returns:
3107             Tuple:
3108                 data from entry
3109                 data from fdtmap (excluding header)
3110                 Image object that was modified
3111         """
3112         dtb_data = self._DoReadFileDtb(dts, use_real_dtb=True,
3113                                        update_dtb=True)[1]
3114
3115         self.assertIn('image', control.images)
3116         image = control.images['image']
3117         entries = image.GetEntries()
3118         orig_dtb_data = entries['u-boot-dtb'].data
3119         orig_fdtmap_data = entries['fdtmap'].data
3120
3121         image_fname = tools.get_output_filename('image.bin')
3122         updated_fname = tools.get_output_filename('image-updated.bin')
3123         tools.write_file(updated_fname, tools.read_file(image_fname))
3124         image = control.WriteEntry(updated_fname, entry_name, data, decomp,
3125                                    allow_resize)
3126         data = control.ReadEntry(updated_fname, entry_name, decomp)
3127
3128         # The DT data should not change unless resized:
3129         if not allow_resize:
3130             new_dtb_data = entries['u-boot-dtb'].data
3131             self.assertEqual(new_dtb_data, orig_dtb_data)
3132             new_fdtmap_data = entries['fdtmap'].data
3133             self.assertEqual(new_fdtmap_data, orig_fdtmap_data)
3134
3135         return data, orig_fdtmap_data[fdtmap.FDTMAP_HDR_LEN:], image
3136
3137     def testReplaceSimple(self):
3138         """Test replacing a single file"""
3139         expected = b'x' * len(U_BOOT_DATA)
3140         data, expected_fdtmap, _ = self._RunReplaceCmd('u-boot', expected,
3141                                                     allow_resize=False)
3142         self.assertEqual(expected, data)
3143
3144         # Test that the state looks right. There should be an FDT for the fdtmap
3145         # that we jsut read back in, and it should match what we find in the
3146         # 'control' tables. Checking for an FDT that does not exist should
3147         # return None.
3148         path, fdtmap = state.GetFdtContents('fdtmap')
3149         self.assertIsNotNone(path)
3150         self.assertEqual(expected_fdtmap, fdtmap)
3151
3152         dtb = state.GetFdtForEtype('fdtmap')
3153         self.assertEqual(dtb.GetContents(), fdtmap)
3154
3155         missing_path, missing_fdtmap = state.GetFdtContents('missing')
3156         self.assertIsNone(missing_path)
3157         self.assertIsNone(missing_fdtmap)
3158
3159         missing_dtb = state.GetFdtForEtype('missing')
3160         self.assertIsNone(missing_dtb)
3161
3162         self.assertEqual('/binman', state.fdt_path_prefix)
3163
3164     def testReplaceResizeFail(self):
3165         """Test replacing a file by something larger"""
3166         expected = U_BOOT_DATA + b'x'
3167         with self.assertRaises(ValueError) as e:
3168             self._RunReplaceCmd('u-boot', expected, allow_resize=False,
3169                                 dts='139_replace_repack.dts')
3170         self.assertIn("Node '/u-boot': Entry data size does not match, but resize is disabled",
3171                       str(e.exception))
3172
3173     def testReplaceMulti(self):
3174         """Test replacing entry data where multiple images are generated"""
3175         data = self._DoReadFileDtb('133_replace_multi.dts', use_real_dtb=True,
3176                                    update_dtb=True)[0]
3177         expected = b'x' * len(U_BOOT_DATA)
3178         updated_fname = tools.get_output_filename('image-updated.bin')
3179         tools.write_file(updated_fname, data)
3180         entry_name = 'u-boot'
3181         control.WriteEntry(updated_fname, entry_name, expected,
3182                            allow_resize=False)
3183         data = control.ReadEntry(updated_fname, entry_name)
3184         self.assertEqual(expected, data)
3185
3186         # Check the state looks right.
3187         self.assertEqual('/binman/image', state.fdt_path_prefix)
3188
3189         # Now check we can write the first image
3190         image_fname = tools.get_output_filename('first-image.bin')
3191         updated_fname = tools.get_output_filename('first-updated.bin')
3192         tools.write_file(updated_fname, tools.read_file(image_fname))
3193         entry_name = 'u-boot'
3194         control.WriteEntry(updated_fname, entry_name, expected,
3195                            allow_resize=False)
3196         data = control.ReadEntry(updated_fname, entry_name)
3197         self.assertEqual(expected, data)
3198
3199         # Check the state looks right.
3200         self.assertEqual('/binman/first-image', state.fdt_path_prefix)
3201
3202     def testUpdateFdtAllRepack(self):
3203         """Test that all device trees are updated with offset/size info"""
3204         data = self._DoReadFileRealDtb('134_fdt_update_all_repack.dts')
3205         SECTION_SIZE = 0x300
3206         DTB_SIZE = 602
3207         FDTMAP_SIZE = 608
3208         base_expected = {
3209             'offset': 0,
3210             'size': SECTION_SIZE + DTB_SIZE * 2 + FDTMAP_SIZE,
3211             'image-pos': 0,
3212             'section:offset': 0,
3213             'section:size': SECTION_SIZE,
3214             'section:image-pos': 0,
3215             'section/u-boot-dtb:offset': 4,
3216             'section/u-boot-dtb:size': 636,
3217             'section/u-boot-dtb:image-pos': 4,
3218             'u-boot-spl-dtb:offset': SECTION_SIZE,
3219             'u-boot-spl-dtb:size': DTB_SIZE,
3220             'u-boot-spl-dtb:image-pos': SECTION_SIZE,
3221             'u-boot-tpl-dtb:offset': SECTION_SIZE + DTB_SIZE,
3222             'u-boot-tpl-dtb:image-pos': SECTION_SIZE + DTB_SIZE,
3223             'u-boot-tpl-dtb:size': DTB_SIZE,
3224             'fdtmap:offset': SECTION_SIZE + DTB_SIZE * 2,
3225             'fdtmap:size': FDTMAP_SIZE,
3226             'fdtmap:image-pos': SECTION_SIZE + DTB_SIZE * 2,
3227         }
3228         main_expected = {
3229             'section:orig-size': SECTION_SIZE,
3230             'section/u-boot-dtb:orig-offset': 4,
3231         }
3232
3233         # We expect three device-tree files in the output, with the first one
3234         # within a fixed-size section.
3235         # Read them in sequence. We look for an 'spl' property in the SPL tree,
3236         # and 'tpl' in the TPL tree, to make sure they are distinct from the
3237         # main U-Boot tree. All three should have the same positions and offset
3238         # except that the main tree should include the main_expected properties
3239         start = 4
3240         for item in ['', 'spl', 'tpl', None]:
3241             if item is None:
3242                 start += 16  # Move past fdtmap header
3243             dtb = fdt.Fdt.FromData(data[start:])
3244             dtb.Scan()
3245             props = self._GetPropTree(dtb,
3246                 BASE_DTB_PROPS + REPACK_DTB_PROPS + ['spl', 'tpl'],
3247                 prefix='/' if item is None else '/binman/')
3248             expected = dict(base_expected)
3249             if item:
3250                 expected[item] = 0
3251             else:
3252                 # Main DTB and fdtdec should include the 'orig-' properties
3253                 expected.update(main_expected)
3254             # Helpful for debugging:
3255             #for prop in sorted(props):
3256                 #print('prop %s %s %s' % (prop, props[prop], expected[prop]))
3257             self.assertEqual(expected, props)
3258             if item == '':
3259                 start = SECTION_SIZE
3260             else:
3261                 start += dtb._fdt_obj.totalsize()
3262
3263     def testFdtmapHeaderMiddle(self):
3264         """Test an FDT map in the middle of an image when it should be at end"""
3265         with self.assertRaises(ValueError) as e:
3266             self._DoReadFileRealDtb('135_fdtmap_hdr_middle.dts')
3267         self.assertIn("Invalid sibling order 'middle' for image-header: Must be at 'end' to match location",
3268                       str(e.exception))
3269
3270     def testFdtmapHeaderStartBad(self):
3271         """Test an FDT map in middle of an image when it should be at start"""
3272         with self.assertRaises(ValueError) as e:
3273             self._DoReadFileRealDtb('136_fdtmap_hdr_startbad.dts')
3274         self.assertIn("Invalid sibling order 'end' for image-header: Must be at 'start' to match location",
3275                       str(e.exception))
3276
3277     def testFdtmapHeaderEndBad(self):
3278         """Test an FDT map at the start of an image when it should be at end"""
3279         with self.assertRaises(ValueError) as e:
3280             self._DoReadFileRealDtb('137_fdtmap_hdr_endbad.dts')
3281         self.assertIn("Invalid sibling order 'start' for image-header: Must be at 'end' to match location",
3282                       str(e.exception))
3283
3284     def testFdtmapHeaderNoSize(self):
3285         """Test an image header at the end of an image with undefined size"""
3286         self._DoReadFileRealDtb('138_fdtmap_hdr_nosize.dts')
3287
3288     def testReplaceResize(self):
3289         """Test replacing a single file in an entry with a larger file"""
3290         expected = U_BOOT_DATA + b'x'
3291         data, _, image = self._RunReplaceCmd('u-boot', expected,
3292                                              dts='139_replace_repack.dts')
3293         self.assertEqual(expected, data)
3294
3295         entries = image.GetEntries()
3296         dtb_data = entries['u-boot-dtb'].data
3297         dtb = fdt.Fdt.FromData(dtb_data)
3298         dtb.Scan()
3299
3300         # The u-boot section should now be larger in the dtb
3301         node = dtb.GetNode('/binman/u-boot')
3302         self.assertEqual(len(expected), fdt_util.GetInt(node, 'size'))
3303
3304         # Same for the fdtmap
3305         fdata = entries['fdtmap'].data
3306         fdtb = fdt.Fdt.FromData(fdata[fdtmap.FDTMAP_HDR_LEN:])
3307         fdtb.Scan()
3308         fnode = fdtb.GetNode('/u-boot')
3309         self.assertEqual(len(expected), fdt_util.GetInt(fnode, 'size'))
3310
3311     def testReplaceResizeNoRepack(self):
3312         """Test replacing an entry with a larger file when not allowed"""
3313         expected = U_BOOT_DATA + b'x'
3314         with self.assertRaises(ValueError) as e:
3315             self._RunReplaceCmd('u-boot', expected)
3316         self.assertIn('Entry data size does not match, but allow-repack is not present for this image',
3317                       str(e.exception))
3318
3319     def testEntryShrink(self):
3320         """Test contracting an entry after it is packed"""
3321         try:
3322             state.SetAllowEntryContraction(True)
3323             data = self._DoReadFileDtb('140_entry_shrink.dts',
3324                                        update_dtb=True)[0]
3325         finally:
3326             state.SetAllowEntryContraction(False)
3327         self.assertEqual(b'a', data[:1])
3328         self.assertEqual(U_BOOT_DATA, data[1:1 + len(U_BOOT_DATA)])
3329         self.assertEqual(b'a', data[-1:])
3330
3331     def testEntryShrinkFail(self):
3332         """Test not being allowed to contract an entry after it is packed"""
3333         data = self._DoReadFileDtb('140_entry_shrink.dts', update_dtb=True)[0]
3334
3335         # In this case there is a spare byte at the end of the data. The size of
3336         # the contents is only 1 byte but we still have the size before it
3337         # shrunk.
3338         self.assertEqual(b'a\0', data[:2])
3339         self.assertEqual(U_BOOT_DATA, data[2:2 + len(U_BOOT_DATA)])
3340         self.assertEqual(b'a\0', data[-2:])
3341
3342     def testDescriptorOffset(self):
3343         """Test that the Intel descriptor is always placed at at the start"""
3344         data = self._DoReadFileDtb('141_descriptor_offset.dts')
3345         image = control.images['image']
3346         entries = image.GetEntries()
3347         desc = entries['intel-descriptor']
3348         self.assertEqual(0xff800000, desc.offset);
3349         self.assertEqual(0xff800000, desc.image_pos);
3350
3351     def testReplaceCbfs(self):
3352         """Test replacing a single file in CBFS without changing the size"""
3353         self._CheckLz4()
3354         expected = b'x' * len(U_BOOT_DATA)
3355         data = self._DoReadFileRealDtb('142_replace_cbfs.dts')
3356         updated_fname = tools.get_output_filename('image-updated.bin')
3357         tools.write_file(updated_fname, data)
3358         entry_name = 'section/cbfs/u-boot'
3359         control.WriteEntry(updated_fname, entry_name, expected,
3360                            allow_resize=True)
3361         data = control.ReadEntry(updated_fname, entry_name)
3362         self.assertEqual(expected, data)
3363
3364     def testReplaceResizeCbfs(self):
3365         """Test replacing a single file in CBFS with one of a different size"""
3366         self._CheckLz4()
3367         expected = U_BOOT_DATA + b'x'
3368         data = self._DoReadFileRealDtb('142_replace_cbfs.dts')
3369         updated_fname = tools.get_output_filename('image-updated.bin')
3370         tools.write_file(updated_fname, data)
3371         entry_name = 'section/cbfs/u-boot'
3372         control.WriteEntry(updated_fname, entry_name, expected,
3373                            allow_resize=True)
3374         data = control.ReadEntry(updated_fname, entry_name)
3375         self.assertEqual(expected, data)
3376
3377     def _SetupForReplace(self):
3378         """Set up some files to use to replace entries
3379
3380         This generates an image, copies it to a new file, extracts all the files
3381         in it and updates some of them
3382
3383         Returns:
3384             List
3385                 Image filename
3386                 Output directory
3387                 Expected values for updated entries, each a string
3388         """
3389         data = self._DoReadFileRealDtb('143_replace_all.dts')
3390
3391         updated_fname = tools.get_output_filename('image-updated.bin')
3392         tools.write_file(updated_fname, data)
3393
3394         outdir = os.path.join(self._indir, 'extract')
3395         einfos = control.ExtractEntries(updated_fname, None, outdir, [])
3396
3397         expected1 = b'x' + U_BOOT_DATA + b'y'
3398         u_boot_fname1 = os.path.join(outdir, 'u-boot')
3399         tools.write_file(u_boot_fname1, expected1)
3400
3401         expected2 = b'a' + U_BOOT_DATA + b'b'
3402         u_boot_fname2 = os.path.join(outdir, 'u-boot2')
3403         tools.write_file(u_boot_fname2, expected2)
3404
3405         expected_text = b'not the same text'
3406         text_fname = os.path.join(outdir, 'text')
3407         tools.write_file(text_fname, expected_text)
3408
3409         dtb_fname = os.path.join(outdir, 'u-boot-dtb')
3410         dtb = fdt.FdtScan(dtb_fname)
3411         node = dtb.GetNode('/binman/text')
3412         node.AddString('my-property', 'the value')
3413         dtb.Sync(auto_resize=True)
3414         dtb.Flush()
3415
3416         return updated_fname, outdir, expected1, expected2, expected_text
3417
3418     def _CheckReplaceMultiple(self, entry_paths):
3419         """Handle replacing the contents of multiple entries
3420
3421         Args:
3422             entry_paths: List of entry paths to replace
3423
3424         Returns:
3425             List
3426                 Dict of entries in the image:
3427                     key: Entry name
3428                     Value: Entry object
3429             Expected values for updated entries, each a string
3430         """
3431         updated_fname, outdir, expected1, expected2, expected_text = (
3432             self._SetupForReplace())
3433         control.ReplaceEntries(updated_fname, None, outdir, entry_paths)
3434
3435         image = Image.FromFile(updated_fname)
3436         image.LoadData()
3437         return image.GetEntries(), expected1, expected2, expected_text
3438
3439     def testReplaceAll(self):
3440         """Test replacing the contents of all entries"""
3441         entries, expected1, expected2, expected_text = (
3442             self._CheckReplaceMultiple([]))
3443         data = entries['u-boot'].data
3444         self.assertEqual(expected1, data)
3445
3446         data = entries['u-boot2'].data
3447         self.assertEqual(expected2, data)
3448
3449         data = entries['text'].data
3450         self.assertEqual(expected_text, data)
3451
3452         # Check that the device tree is updated
3453         data = entries['u-boot-dtb'].data
3454         dtb = fdt.Fdt.FromData(data)
3455         dtb.Scan()
3456         node = dtb.GetNode('/binman/text')
3457         self.assertEqual('the value', node.props['my-property'].value)
3458
3459     def testReplaceSome(self):
3460         """Test replacing the contents of a few entries"""
3461         entries, expected1, expected2, expected_text = (
3462             self._CheckReplaceMultiple(['u-boot2', 'text']))
3463
3464         # This one should not change
3465         data = entries['u-boot'].data
3466         self.assertEqual(U_BOOT_DATA, data)
3467
3468         data = entries['u-boot2'].data
3469         self.assertEqual(expected2, data)
3470
3471         data = entries['text'].data
3472         self.assertEqual(expected_text, data)
3473
3474     def testReplaceCmd(self):
3475         """Test replacing a file fron an image on the command line"""
3476         self._DoReadFileRealDtb('143_replace_all.dts')
3477
3478         try:
3479             tmpdir, updated_fname = self._SetupImageInTmpdir()
3480
3481             fname = os.path.join(tmpdir, 'update-u-boot.bin')
3482             expected = b'x' * len(U_BOOT_DATA)
3483             tools.write_file(fname, expected)
3484
3485             self._DoBinman('replace', '-i', updated_fname, 'u-boot', '-f', fname)
3486             data = tools.read_file(updated_fname)
3487             self.assertEqual(expected, data[:len(expected)])
3488             map_fname = os.path.join(tmpdir, 'image-updated.map')
3489             self.assertFalse(os.path.exists(map_fname))
3490         finally:
3491             shutil.rmtree(tmpdir)
3492
3493     def testReplaceCmdSome(self):
3494         """Test replacing some files fron an image on the command line"""
3495         updated_fname, outdir, expected1, expected2, expected_text = (
3496             self._SetupForReplace())
3497
3498         self._DoBinman('replace', '-i', updated_fname, '-I', outdir,
3499                        'u-boot2', 'text')
3500
3501         tools.prepare_output_dir(None)
3502         image = Image.FromFile(updated_fname)
3503         image.LoadData()
3504         entries = image.GetEntries()
3505
3506         # This one should not change
3507         data = entries['u-boot'].data
3508         self.assertEqual(U_BOOT_DATA, data)
3509
3510         data = entries['u-boot2'].data
3511         self.assertEqual(expected2, data)
3512
3513         data = entries['text'].data
3514         self.assertEqual(expected_text, data)
3515
3516     def testReplaceMissing(self):
3517         """Test replacing entries where the file is missing"""
3518         updated_fname, outdir, expected1, expected2, expected_text = (
3519             self._SetupForReplace())
3520
3521         # Remove one of the files, to generate a warning
3522         u_boot_fname1 = os.path.join(outdir, 'u-boot')
3523         os.remove(u_boot_fname1)
3524
3525         with test_util.capture_sys_output() as (stdout, stderr):
3526             control.ReplaceEntries(updated_fname, None, outdir, [])
3527         self.assertIn("Skipping entry '/u-boot' from missing file",
3528                       stderr.getvalue())
3529
3530     def testReplaceCmdMap(self):
3531         """Test replacing a file fron an image on the command line"""
3532         self._DoReadFileRealDtb('143_replace_all.dts')
3533
3534         try:
3535             tmpdir, updated_fname = self._SetupImageInTmpdir()
3536
3537             fname = os.path.join(self._indir, 'update-u-boot.bin')
3538             expected = b'x' * len(U_BOOT_DATA)
3539             tools.write_file(fname, expected)
3540
3541             self._DoBinman('replace', '-i', updated_fname, 'u-boot',
3542                            '-f', fname, '-m')
3543             map_fname = os.path.join(tmpdir, 'image-updated.map')
3544             self.assertTrue(os.path.exists(map_fname))
3545         finally:
3546             shutil.rmtree(tmpdir)
3547
3548     def testReplaceNoEntryPaths(self):
3549         """Test replacing an entry without an entry path"""
3550         self._DoReadFileRealDtb('143_replace_all.dts')
3551         image_fname = tools.get_output_filename('image.bin')
3552         with self.assertRaises(ValueError) as e:
3553             control.ReplaceEntries(image_fname, 'fname', None, [])
3554         self.assertIn('Must specify an entry path to read with -f',
3555                       str(e.exception))
3556
3557     def testReplaceTooManyEntryPaths(self):
3558         """Test extracting some entries"""
3559         self._DoReadFileRealDtb('143_replace_all.dts')
3560         image_fname = tools.get_output_filename('image.bin')
3561         with self.assertRaises(ValueError) as e:
3562             control.ReplaceEntries(image_fname, 'fname', None, ['a', 'b'])
3563         self.assertIn('Must specify exactly one entry path to write with -f',
3564                       str(e.exception))
3565
3566     def testPackReset16(self):
3567         """Test that an image with an x86 reset16 region can be created"""
3568         data = self._DoReadFile('144_x86_reset16.dts')
3569         self.assertEqual(X86_RESET16_DATA, data[:len(X86_RESET16_DATA)])
3570
3571     def testPackReset16Spl(self):
3572         """Test that an image with an x86 reset16-spl region can be created"""
3573         data = self._DoReadFile('145_x86_reset16_spl.dts')
3574         self.assertEqual(X86_RESET16_SPL_DATA, data[:len(X86_RESET16_SPL_DATA)])
3575
3576     def testPackReset16Tpl(self):
3577         """Test that an image with an x86 reset16-tpl region can be created"""
3578         data = self._DoReadFile('146_x86_reset16_tpl.dts')
3579         self.assertEqual(X86_RESET16_TPL_DATA, data[:len(X86_RESET16_TPL_DATA)])
3580
3581     def testPackIntelFit(self):
3582         """Test that an image with an Intel FIT and pointer can be created"""
3583         data = self._DoReadFile('147_intel_fit.dts')
3584         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
3585         fit = data[16:32];
3586         self.assertEqual(b'_FIT_   \x01\x00\x00\x00\x00\x01\x80}' , fit)
3587         ptr = struct.unpack('<i', data[0x40:0x44])[0]
3588
3589         image = control.images['image']
3590         entries = image.GetEntries()
3591         expected_ptr = entries['intel-fit'].image_pos - (1 << 32)
3592         self.assertEqual(expected_ptr, ptr)
3593
3594     def testPackIntelFitMissing(self):
3595         """Test detection of a FIT pointer with not FIT region"""
3596         with self.assertRaises(ValueError) as e:
3597             self._DoReadFile('148_intel_fit_missing.dts')
3598         self.assertIn("'intel-fit-ptr' section must have an 'intel-fit' sibling",
3599                       str(e.exception))
3600
3601     def _CheckSymbolsTplSection(self, dts, expected_vals):
3602         data = self._DoReadFile(dts)
3603         sym_values = struct.pack('<LQLL', *expected_vals)
3604         upto1 = 4 + len(U_BOOT_SPL_DATA)
3605         expected1 = tools.get_bytes(0xff, 4) + sym_values + U_BOOT_SPL_DATA[20:]
3606         self.assertEqual(expected1, data[:upto1])
3607
3608         upto2 = upto1 + 1 + len(U_BOOT_SPL_DATA)
3609         expected2 = tools.get_bytes(0xff, 1) + sym_values + U_BOOT_SPL_DATA[20:]
3610         self.assertEqual(expected2, data[upto1:upto2])
3611
3612         upto3 = 0x34 + len(U_BOOT_DATA)
3613         expected3 = tools.get_bytes(0xff, 1) + U_BOOT_DATA
3614         self.assertEqual(expected3, data[upto2:upto3])
3615
3616         expected4 = sym_values + U_BOOT_TPL_DATA[20:]
3617         self.assertEqual(expected4, data[upto3:upto3 + len(U_BOOT_TPL_DATA)])
3618
3619     def testSymbolsTplSection(self):
3620         """Test binman can assign symbols embedded in U-Boot TPL in a section"""
3621         self._SetupSplElf('u_boot_binman_syms')
3622         self._SetupTplElf('u_boot_binman_syms')
3623         self._CheckSymbolsTplSection('149_symbols_tpl.dts',
3624                                      [0x04, 0x1c, 0x10 + 0x34, 0x04])
3625
3626     def testSymbolsTplSectionX86(self):
3627         """Test binman can assign symbols in a section with end-at-4gb"""
3628         self._SetupSplElf('u_boot_binman_syms_x86')
3629         self._SetupTplElf('u_boot_binman_syms_x86')
3630         self._CheckSymbolsTplSection('155_symbols_tpl_x86.dts',
3631                                      [0xffffff04, 0xffffff1c, 0xffffff34,
3632                                       0x04])
3633
3634     def testPackX86RomIfwiSectiom(self):
3635         """Test that a section can be placed in an IFWI region"""
3636         self._SetupIfwi('fitimage.bin')
3637         data = self._DoReadFile('151_x86_rom_ifwi_section.dts')
3638         self._CheckIfwi(data)
3639
3640     def testPackFspM(self):
3641         """Test that an image with a FSP memory-init binary can be created"""
3642         data = self._DoReadFile('152_intel_fsp_m.dts')
3643         self.assertEqual(FSP_M_DATA, data[:len(FSP_M_DATA)])
3644
3645     def testPackFspS(self):
3646         """Test that an image with a FSP silicon-init binary can be created"""
3647         data = self._DoReadFile('153_intel_fsp_s.dts')
3648         self.assertEqual(FSP_S_DATA, data[:len(FSP_S_DATA)])
3649
3650     def testPackFspT(self):
3651         """Test that an image with a FSP temp-ram-init binary can be created"""
3652         data = self._DoReadFile('154_intel_fsp_t.dts')
3653         self.assertEqual(FSP_T_DATA, data[:len(FSP_T_DATA)])
3654
3655     def testMkimage(self):
3656         """Test using mkimage to build an image"""
3657         data = self._DoReadFile('156_mkimage.dts')
3658
3659         # Just check that the data appears in the file somewhere
3660         self.assertIn(U_BOOT_SPL_DATA, data)
3661
3662     def testMkimageMissing(self):
3663         """Test that binman still produces an image if mkimage is missing"""
3664         with test_util.capture_sys_output() as (_, stderr):
3665             self._DoTestFile('156_mkimage.dts',
3666                              force_missing_bintools='mkimage')
3667         err = stderr.getvalue()
3668         self.assertRegex(err,
3669                          "Image 'main-section'.*missing bintools.*: mkimage")
3670
3671     def testExtblob(self):
3672         """Test an image with an external blob"""
3673         data = self._DoReadFile('157_blob_ext.dts')
3674         self.assertEqual(REFCODE_DATA, data)
3675
3676     def testExtblobMissing(self):
3677         """Test an image with a missing external blob"""
3678         with self.assertRaises(ValueError) as e:
3679             self._DoReadFile('158_blob_ext_missing.dts')
3680         self.assertIn("Filename 'missing-file' not found in input path",
3681                       str(e.exception))
3682
3683     def testExtblobMissingOk(self):
3684         """Test an image with an missing external blob that is allowed"""
3685         with test_util.capture_sys_output() as (stdout, stderr):
3686             self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True)
3687         err = stderr.getvalue()
3688         self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext")
3689
3690     def testExtblobMissingOkSect(self):
3691         """Test an image with an missing external blob that is allowed"""
3692         with test_util.capture_sys_output() as (stdout, stderr):
3693             self._DoTestFile('159_blob_ext_missing_sect.dts',
3694                              allow_missing=True)
3695         err = stderr.getvalue()
3696         self.assertRegex(err, "Image 'main-section'.*missing.*: "
3697                          "blob-ext blob-ext2")
3698
3699     def testPackX86RomMeMissingDesc(self):
3700         """Test that an missing Intel descriptor entry is allowed"""
3701         with test_util.capture_sys_output() as (stdout, stderr):
3702             self._DoTestFile('164_x86_rom_me_missing.dts', allow_missing=True)
3703         err = stderr.getvalue()
3704         self.assertRegex(err,
3705                          "Image 'main-section'.*missing.*: intel-descriptor")
3706
3707     def testPackX86RomMissingIfwi(self):
3708         """Test that an x86 ROM with Integrated Firmware Image can be created"""
3709         self._SetupIfwi('fitimage.bin')
3710         pathname = os.path.join(self._indir, 'fitimage.bin')
3711         os.remove(pathname)
3712         with test_util.capture_sys_output() as (stdout, stderr):
3713             self._DoTestFile('111_x86_rom_ifwi.dts', allow_missing=True)
3714         err = stderr.getvalue()
3715         self.assertRegex(err, "Image 'main-section'.*missing.*: intel-ifwi")
3716
3717     def testPackOverlapZero(self):
3718         """Test that zero-size overlapping regions are ignored"""
3719         self._DoTestFile('160_pack_overlap_zero.dts')
3720
3721     def _CheckSimpleFitData(self, fit_data, kernel_data, fdt1_data):
3722         # The data should be inside the FIT
3723         dtb = fdt.Fdt.FromData(fit_data)
3724         dtb.Scan()
3725         fnode = dtb.GetNode('/images/kernel')
3726         self.assertIn('data', fnode.props)
3727
3728         fname = os.path.join(self._indir, 'fit_data.fit')
3729         tools.write_file(fname, fit_data)
3730         out = tools.run('dumpimage', '-l', fname)
3731
3732         # Check a few features to make sure the plumbing works. We don't need
3733         # to test the operation of mkimage or dumpimage here. First convert the
3734         # output into a dict where the keys are the fields printed by dumpimage
3735         # and the values are a list of values for each field
3736         lines = out.splitlines()
3737
3738         # Converts "Compression:  gzip compressed" into two groups:
3739         # 'Compression' and 'gzip compressed'
3740         re_line = re.compile(r'^ *([^:]*)(?:: *(.*))?$')
3741         vals = collections.defaultdict(list)
3742         for line in lines:
3743             mat = re_line.match(line)
3744             vals[mat.group(1)].append(mat.group(2))
3745
3746         self.assertEquals('FIT description: test-desc', lines[0])
3747         self.assertIn('Created:', lines[1])
3748         self.assertIn('Image 0 (kernel)', vals)
3749         self.assertIn('Hash value', vals)
3750         data_sizes = vals.get('Data Size')
3751         self.assertIsNotNone(data_sizes)
3752         self.assertEqual(2, len(data_sizes))
3753         # Format is "4 Bytes = 0.00 KiB = 0.00 MiB" so take the first word
3754         self.assertEqual(len(kernel_data), int(data_sizes[0].split()[0]))
3755         self.assertEqual(len(fdt1_data), int(data_sizes[1].split()[0]))
3756
3757     def testSimpleFit(self):
3758         """Test an image with a FIT inside"""
3759         data = self._DoReadFile('161_fit.dts')
3760         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
3761         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
3762         fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
3763
3764         self._CheckSimpleFitData(fit_data, U_BOOT_DATA, U_BOOT_SPL_DTB_DATA)
3765
3766     def testSimpleFitExpandsSubentries(self):
3767         """Test that FIT images expand their subentries"""
3768         data = self._DoReadFileDtb('161_fit.dts', use_expanded=True)[0]
3769         self.assertEqual(U_BOOT_EXP_DATA, data[:len(U_BOOT_EXP_DATA)])
3770         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
3771         fit_data = data[len(U_BOOT_EXP_DATA):-len(U_BOOT_NODTB_DATA)]
3772
3773         self._CheckSimpleFitData(fit_data, U_BOOT_EXP_DATA, U_BOOT_SPL_DTB_DATA)
3774
3775     def testSimpleFitImagePos(self):
3776         """Test that we have correct image-pos for FIT subentries"""
3777         data, _, _, out_dtb_fname = self._DoReadFileDtb('161_fit.dts',
3778                                                         update_dtb=True)
3779         dtb = fdt.Fdt(out_dtb_fname)
3780         dtb.Scan()
3781         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
3782
3783         self.maxDiff = None
3784         self.assertEqual({
3785             'image-pos': 0,
3786             'offset': 0,
3787             'size': 1890,
3788
3789             'u-boot:image-pos': 0,
3790             'u-boot:offset': 0,
3791             'u-boot:size': 4,
3792
3793             'fit:image-pos': 4,
3794             'fit:offset': 4,
3795             'fit:size': 1840,
3796
3797             'fit/images/kernel:image-pos': 304,
3798             'fit/images/kernel:offset': 300,
3799             'fit/images/kernel:size': 4,
3800
3801             'fit/images/kernel/u-boot:image-pos': 304,
3802             'fit/images/kernel/u-boot:offset': 0,
3803             'fit/images/kernel/u-boot:size': 4,
3804
3805             'fit/images/fdt-1:image-pos': 552,
3806             'fit/images/fdt-1:offset': 548,
3807             'fit/images/fdt-1:size': 6,
3808
3809             'fit/images/fdt-1/u-boot-spl-dtb:image-pos': 552,
3810             'fit/images/fdt-1/u-boot-spl-dtb:offset': 0,
3811             'fit/images/fdt-1/u-boot-spl-dtb:size': 6,
3812
3813             'u-boot-nodtb:image-pos': 1844,
3814             'u-boot-nodtb:offset': 1844,
3815             'u-boot-nodtb:size': 46,
3816         }, props)
3817
3818         # Actually check the data is where we think it is
3819         for node, expected in [
3820             ("u-boot", U_BOOT_DATA),
3821             ("fit/images/kernel", U_BOOT_DATA),
3822             ("fit/images/kernel/u-boot", U_BOOT_DATA),
3823             ("fit/images/fdt-1", U_BOOT_SPL_DTB_DATA),
3824             ("fit/images/fdt-1/u-boot-spl-dtb", U_BOOT_SPL_DTB_DATA),
3825             ("u-boot-nodtb", U_BOOT_NODTB_DATA),
3826         ]:
3827             image_pos = props[f"{node}:image-pos"]
3828             size = props[f"{node}:size"]
3829             self.assertEqual(len(expected), size)
3830             self.assertEqual(expected, data[image_pos:image_pos+size])
3831
3832     def testFitExternal(self):
3833         """Test an image with an FIT with external images"""
3834         data = self._DoReadFile('162_fit_external.dts')
3835         fit_data = data[len(U_BOOT_DATA):-2]  # _testing is 2 bytes
3836
3837         # Size of the external-data region as set up by mkimage
3838         external_data_size = len(U_BOOT_DATA) + 2
3839         expected_size = (len(U_BOOT_DATA) + 0x400 +
3840                          tools.align(external_data_size, 4) +
3841                          len(U_BOOT_NODTB_DATA))
3842
3843         # The data should be outside the FIT
3844         dtb = fdt.Fdt.FromData(fit_data)
3845         dtb.Scan()
3846         fnode = dtb.GetNode('/images/kernel')
3847         self.assertNotIn('data', fnode.props)
3848         self.assertEqual(len(U_BOOT_DATA),
3849                          fdt_util.fdt32_to_cpu(fnode.props['data-size'].value))
3850         fit_pos = 0x400;
3851         self.assertEqual(
3852             fit_pos,
3853             fdt_util.fdt32_to_cpu(fnode.props['data-position'].value))
3854
3855         self.assertEquals(expected_size, len(data))
3856         actual_pos = len(U_BOOT_DATA) + fit_pos
3857         self.assertEqual(U_BOOT_DATA + b'aa',
3858                          data[actual_pos:actual_pos + external_data_size])
3859
3860     def testFitExternalImagePos(self):
3861         """Test that we have correct image-pos for external FIT subentries"""
3862         data, _, _, out_dtb_fname = self._DoReadFileDtb('162_fit_external.dts',
3863                                                         update_dtb=True)
3864         dtb = fdt.Fdt(out_dtb_fname)
3865         dtb.Scan()
3866         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
3867
3868         self.assertEqual({
3869             'image-pos': 0,
3870             'offset': 0,
3871             'size': 1082,
3872
3873             'u-boot:image-pos': 0,
3874             'u-boot:offset': 0,
3875             'u-boot:size': 4,
3876
3877             'fit:size': 1032,
3878             'fit:offset': 4,
3879             'fit:image-pos': 4,
3880
3881             'fit/images/kernel:size': 4,
3882             'fit/images/kernel:offset': 1024,
3883             'fit/images/kernel:image-pos': 1028,
3884
3885             'fit/images/kernel/u-boot:size': 4,
3886             'fit/images/kernel/u-boot:offset': 0,
3887             'fit/images/kernel/u-boot:image-pos': 1028,
3888
3889             'fit/images/fdt-1:size': 2,
3890             'fit/images/fdt-1:offset': 1028,
3891             'fit/images/fdt-1:image-pos': 1032,
3892
3893             'fit/images/fdt-1/_testing:size': 2,
3894             'fit/images/fdt-1/_testing:offset': 0,
3895             'fit/images/fdt-1/_testing:image-pos': 1032,
3896
3897             'u-boot-nodtb:image-pos': 1036,
3898             'u-boot-nodtb:offset': 1036,
3899             'u-boot-nodtb:size': 46,
3900          }, props)
3901
3902         # Actually check the data is where we think it is
3903         for node, expected in [
3904             ("u-boot", U_BOOT_DATA),
3905             ("fit/images/kernel", U_BOOT_DATA),
3906             ("fit/images/kernel/u-boot", U_BOOT_DATA),
3907             ("fit/images/fdt-1", b'aa'),
3908             ("fit/images/fdt-1/_testing", b'aa'),
3909             ("u-boot-nodtb", U_BOOT_NODTB_DATA),
3910         ]:
3911             image_pos = props[f"{node}:image-pos"]
3912             size = props[f"{node}:size"]
3913             self.assertEqual(len(expected), size)
3914             self.assertEqual(expected, data[image_pos:image_pos+size])
3915
3916     def testFitMissing(self):
3917         """Test that binman still produces a FIT image if mkimage is missing"""
3918         with test_util.capture_sys_output() as (_, stderr):
3919             self._DoTestFile('162_fit_external.dts',
3920                              force_missing_bintools='mkimage')
3921         err = stderr.getvalue()
3922         self.assertRegex(err,
3923                          "Image 'main-section'.*missing bintools.*: mkimage")
3924
3925     def testSectionIgnoreHashSignature(self):
3926         """Test that sections ignore hash, signature nodes for its data"""
3927         data = self._DoReadFile('165_section_ignore_hash_signature.dts')
3928         expected = (U_BOOT_DATA + U_BOOT_DATA)
3929         self.assertEqual(expected, data)
3930
3931     def testPadInSections(self):
3932         """Test pad-before, pad-after for entries in sections"""
3933         data, _, _, out_dtb_fname = self._DoReadFileDtb(
3934             '166_pad_in_sections.dts', update_dtb=True)
3935         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
3936                     U_BOOT_DATA + tools.get_bytes(ord('!'), 6) +
3937                     U_BOOT_DATA)
3938         self.assertEqual(expected, data)
3939
3940         dtb = fdt.Fdt(out_dtb_fname)
3941         dtb.Scan()
3942         props = self._GetPropTree(dtb, ['size', 'image-pos', 'offset'])
3943         expected = {
3944             'image-pos': 0,
3945             'offset': 0,
3946             'size': 12 + 6 + 3 * len(U_BOOT_DATA),
3947
3948             'section:image-pos': 0,
3949             'section:offset': 0,
3950             'section:size': 12 + 6 + 3 * len(U_BOOT_DATA),
3951
3952             'section/before:image-pos': 0,
3953             'section/before:offset': 0,
3954             'section/before:size': len(U_BOOT_DATA),
3955
3956             'section/u-boot:image-pos': 4,
3957             'section/u-boot:offset': 4,
3958             'section/u-boot:size': 12 + len(U_BOOT_DATA) + 6,
3959
3960             'section/after:image-pos': 26,
3961             'section/after:offset': 26,
3962             'section/after:size': len(U_BOOT_DATA),
3963             }
3964         self.assertEqual(expected, props)
3965
3966     def testFitImageSubentryAlignment(self):
3967         """Test relative alignability of FIT image subentries"""
3968         self._SetupSplElf()
3969         entry_args = {
3970             'test-id': TEXT_DATA,
3971         }
3972         data, _, _, _ = self._DoReadFileDtb('167_fit_image_subentry_alignment.dts',
3973                                             entry_args=entry_args)
3974         dtb = fdt.Fdt.FromData(data)
3975         dtb.Scan()
3976
3977         node = dtb.GetNode('/images/kernel')
3978         data = dtb.GetProps(node)["data"].bytes
3979         align_pad = 0x10 - (len(U_BOOT_SPL_DATA) % 0x10)
3980         expected = (tools.get_bytes(0, 0x20) + U_BOOT_SPL_DATA +
3981                     tools.get_bytes(0, align_pad) + U_BOOT_DATA)
3982         self.assertEqual(expected, data)
3983
3984         node = dtb.GetNode('/images/fdt-1')
3985         data = dtb.GetProps(node)["data"].bytes
3986         expected = (U_BOOT_SPL_DTB_DATA + tools.get_bytes(0, 20) +
3987                     tools.to_bytes(TEXT_DATA) + tools.get_bytes(0, 30) +
3988                     U_BOOT_DTB_DATA)
3989         self.assertEqual(expected, data)
3990
3991     def testFitExtblobMissingOk(self):
3992         """Test a FIT with a missing external blob that is allowed"""
3993         with test_util.capture_sys_output() as (stdout, stderr):
3994             self._DoTestFile('168_fit_missing_blob.dts',
3995                              allow_missing=True)
3996         err = stderr.getvalue()
3997         self.assertRegex(err, "Image 'main-section'.*missing.*: atf-bl31")
3998
3999     def testBlobNamedByArgMissing(self):
4000         """Test handling of a missing entry arg"""
4001         with self.assertRaises(ValueError) as e:
4002             self._DoReadFile('068_blob_named_by_arg.dts')
4003         self.assertIn("Missing required properties/entry args: cros-ec-rw-path",
4004                       str(e.exception))
4005
4006     def testPackBl31(self):
4007         """Test that an image with an ATF BL31 binary can be created"""
4008         data = self._DoReadFile('169_atf_bl31.dts')
4009         self.assertEqual(ATF_BL31_DATA, data[:len(ATF_BL31_DATA)])
4010
4011     def testPackScp(self):
4012         """Test that an image with an SCP binary can be created"""
4013         data = self._DoReadFile('172_scp.dts')
4014         self.assertEqual(SCP_DATA, data[:len(SCP_DATA)])
4015
4016     def testFitFdt(self):
4017         """Test an image with an FIT with multiple FDT images"""
4018         def _CheckFdt(seq, expected_data):
4019             """Check the FDT nodes
4020
4021             Args:
4022                 seq: Sequence number to check (0 or 1)
4023                 expected_data: Expected contents of 'data' property
4024             """
4025             name = 'fdt-%d' % seq
4026             fnode = dtb.GetNode('/images/%s' % name)
4027             self.assertIsNotNone(fnode)
4028             self.assertEqual({'description','type', 'compression', 'data'},
4029                              set(fnode.props.keys()))
4030             self.assertEqual(expected_data, fnode.props['data'].bytes)
4031             self.assertEqual('fdt-test-fdt%d.dtb' % seq,
4032                              fnode.props['description'].value)
4033             self.assertEqual(fnode.subnodes[0].name, 'hash')
4034
4035         def _CheckConfig(seq, expected_data):
4036             """Check the configuration nodes
4037
4038             Args:
4039                 seq: Sequence number to check (0 or 1)
4040                 expected_data: Expected contents of 'data' property
4041             """
4042             cnode = dtb.GetNode('/configurations')
4043             self.assertIn('default', cnode.props)
4044             self.assertEqual('config-2', cnode.props['default'].value)
4045
4046             name = 'config-%d' % seq
4047             fnode = dtb.GetNode('/configurations/%s' % name)
4048             self.assertIsNotNone(fnode)
4049             self.assertEqual({'description','firmware', 'loadables', 'fdt'},
4050                              set(fnode.props.keys()))
4051             self.assertEqual('conf-test-fdt%d.dtb' % seq,
4052                              fnode.props['description'].value)
4053             self.assertEqual('fdt-%d' % seq, fnode.props['fdt'].value)
4054
4055         entry_args = {
4056             'of-list': 'test-fdt1 test-fdt2',
4057             'default-dt': 'test-fdt2',
4058         }
4059         data = self._DoReadFileDtb(
4060             '170_fit_fdt.dts',
4061             entry_args=entry_args,
4062             extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
4063         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
4064         fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
4065
4066         dtb = fdt.Fdt.FromData(fit_data)
4067         dtb.Scan()
4068         fnode = dtb.GetNode('/images/kernel')
4069         self.assertIn('data', fnode.props)
4070
4071         # Check all the properties in fdt-1 and fdt-2
4072         _CheckFdt(1, TEST_FDT1_DATA)
4073         _CheckFdt(2, TEST_FDT2_DATA)
4074
4075         # Check configurations
4076         _CheckConfig(1, TEST_FDT1_DATA)
4077         _CheckConfig(2, TEST_FDT2_DATA)
4078
4079     def testFitFdtMissingList(self):
4080         """Test handling of a missing 'of-list' entry arg"""
4081         with self.assertRaises(ValueError) as e:
4082             self._DoReadFile('170_fit_fdt.dts')
4083         self.assertIn("Generator node requires 'of-list' entry argument",
4084                       str(e.exception))
4085
4086     def testFitFdtEmptyList(self):
4087         """Test handling of an empty 'of-list' entry arg"""
4088         entry_args = {
4089             'of-list': '',
4090         }
4091         data = self._DoReadFileDtb('170_fit_fdt.dts', entry_args=entry_args)[0]
4092
4093     def testFitFdtMissingProp(self):
4094         """Test handling of a missing 'fit,fdt-list' property"""
4095         with self.assertRaises(ValueError) as e:
4096             self._DoReadFile('171_fit_fdt_missing_prop.dts')
4097         self.assertIn("Generator node requires 'fit,fdt-list' property",
4098                       str(e.exception))
4099
4100     def testFitFdtMissing(self):
4101         """Test handling of a missing 'default-dt' entry arg"""
4102         entry_args = {
4103             'of-list': 'test-fdt1 test-fdt2',
4104         }
4105         with self.assertRaises(ValueError) as e:
4106             self._DoReadFileDtb(
4107                 '170_fit_fdt.dts',
4108                 entry_args=entry_args,
4109                 extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
4110         self.assertIn("Generated 'default' node requires default-dt entry argument",
4111                       str(e.exception))
4112
4113     def testFitFdtNotInList(self):
4114         """Test handling of a default-dt that is not in the of-list"""
4115         entry_args = {
4116             'of-list': 'test-fdt1 test-fdt2',
4117             'default-dt': 'test-fdt3',
4118         }
4119         with self.assertRaises(ValueError) as e:
4120             self._DoReadFileDtb(
4121                 '170_fit_fdt.dts',
4122                 entry_args=entry_args,
4123                 extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
4124         self.assertIn("default-dt entry argument 'test-fdt3' not found in fdt list: test-fdt1, test-fdt2",
4125                       str(e.exception))
4126
4127     def testFitExtblobMissingHelp(self):
4128         """Test display of help messages when an external blob is missing"""
4129         control.missing_blob_help = control._ReadMissingBlobHelp()
4130         control.missing_blob_help['wibble'] = 'Wibble test'
4131         control.missing_blob_help['another'] = 'Another test'
4132         with test_util.capture_sys_output() as (stdout, stderr):
4133             self._DoTestFile('168_fit_missing_blob.dts',
4134                              allow_missing=True)
4135         err = stderr.getvalue()
4136
4137         # We can get the tag from the name, the type or the missing-msg
4138         # property. Check all three.
4139         self.assertIn('You may need to build ARM Trusted', err)
4140         self.assertIn('Wibble test', err)
4141         self.assertIn('Another test', err)
4142
4143     def testMissingBlob(self):
4144         """Test handling of a blob containing a missing file"""
4145         with self.assertRaises(ValueError) as e:
4146             self._DoTestFile('173_missing_blob.dts', allow_missing=True)
4147         self.assertIn("Filename 'missing' not found in input path",
4148                       str(e.exception))
4149
4150     def testEnvironment(self):
4151         """Test adding a U-Boot environment"""
4152         data = self._DoReadFile('174_env.dts')
4153         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
4154         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
4155         env = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
4156         self.assertEqual(b'\x1b\x97\x22\x7c\x01var1=1\0var2="2"\0\0\xff\xff',
4157                          env)
4158
4159     def testEnvironmentNoSize(self):
4160         """Test that a missing 'size' property is detected"""
4161         with self.assertRaises(ValueError) as e:
4162             self._DoTestFile('175_env_no_size.dts')
4163         self.assertIn("'u-boot-env' entry must have a size property",
4164                       str(e.exception))
4165
4166     def testEnvironmentTooSmall(self):
4167         """Test handling of an environment that does not fit"""
4168         with self.assertRaises(ValueError) as e:
4169             self._DoTestFile('176_env_too_small.dts')
4170
4171         # checksum, start byte, environment with \0 terminator, final \0
4172         need = 4 + 1 + len(ENV_DATA) + 1 + 1
4173         short = need - 0x8
4174         self.assertIn("too small to hold data (need %#x more bytes)" % short,
4175                       str(e.exception))
4176
4177     def testSkipAtStart(self):
4178         """Test handling of skip-at-start section"""
4179         data = self._DoReadFile('177_skip_at_start.dts')
4180         self.assertEqual(U_BOOT_DATA, data)
4181
4182         image = control.images['image']
4183         entries = image.GetEntries()
4184         section = entries['section']
4185         self.assertEqual(0, section.offset)
4186         self.assertEqual(len(U_BOOT_DATA), section.size)
4187         self.assertEqual(U_BOOT_DATA, section.GetData())
4188
4189         entry = section.GetEntries()['u-boot']
4190         self.assertEqual(16, entry.offset)
4191         self.assertEqual(len(U_BOOT_DATA), entry.size)
4192         self.assertEqual(U_BOOT_DATA, entry.data)
4193
4194     def testSkipAtStartPad(self):
4195         """Test handling of skip-at-start section with padded entry"""
4196         data = self._DoReadFile('178_skip_at_start_pad.dts')
4197         before = tools.get_bytes(0, 8)
4198         after = tools.get_bytes(0, 4)
4199         all = before + U_BOOT_DATA + after
4200         self.assertEqual(all, data)
4201
4202         image = control.images['image']
4203         entries = image.GetEntries()
4204         section = entries['section']
4205         self.assertEqual(0, section.offset)
4206         self.assertEqual(len(all), section.size)
4207         self.assertEqual(all, section.GetData())
4208
4209         entry = section.GetEntries()['u-boot']
4210         self.assertEqual(16, entry.offset)
4211         self.assertEqual(len(all), entry.size)
4212         self.assertEqual(U_BOOT_DATA, entry.data)
4213
4214     def testSkipAtStartSectionPad(self):
4215         """Test handling of skip-at-start section with padding"""
4216         data = self._DoReadFile('179_skip_at_start_section_pad.dts')
4217         before = tools.get_bytes(0, 8)
4218         after = tools.get_bytes(0, 4)
4219         all = before + U_BOOT_DATA + after
4220         self.assertEqual(all, data)
4221
4222         image = control.images['image']
4223         entries = image.GetEntries()
4224         section = entries['section']
4225         self.assertEqual(0, section.offset)
4226         self.assertEqual(len(all), section.size)
4227         self.assertEqual(U_BOOT_DATA, section.data)
4228         self.assertEqual(all, section.GetPaddedData())
4229
4230         entry = section.GetEntries()['u-boot']
4231         self.assertEqual(16, entry.offset)
4232         self.assertEqual(len(U_BOOT_DATA), entry.size)
4233         self.assertEqual(U_BOOT_DATA, entry.data)
4234
4235     def testSectionPad(self):
4236         """Testing padding with sections"""
4237         data = self._DoReadFile('180_section_pad.dts')
4238         expected = (tools.get_bytes(ord('&'), 3) +
4239                     tools.get_bytes(ord('!'), 5) +
4240                     U_BOOT_DATA +
4241                     tools.get_bytes(ord('!'), 1) +
4242                     tools.get_bytes(ord('&'), 2))
4243         self.assertEqual(expected, data)
4244
4245     def testSectionAlign(self):
4246         """Testing alignment with sections"""
4247         data = self._DoReadFileDtb('181_section_align.dts', map=True)[0]
4248         expected = (b'\0' +                         # fill section
4249                     tools.get_bytes(ord('&'), 1) +   # padding to section align
4250                     b'\0' +                         # fill section
4251                     tools.get_bytes(ord('!'), 3) +   # padding to u-boot align
4252                     U_BOOT_DATA +
4253                     tools.get_bytes(ord('!'), 4) +   # padding to u-boot size
4254                     tools.get_bytes(ord('!'), 4))    # padding to section size
4255         self.assertEqual(expected, data)
4256
4257     def testCompressImage(self):
4258         """Test compression of the entire image"""
4259         self._CheckLz4()
4260         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4261             '182_compress_image.dts', use_real_dtb=True, update_dtb=True)
4262         dtb = fdt.Fdt(out_dtb_fname)
4263         dtb.Scan()
4264         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4265                                         'uncomp-size'])
4266         orig = self._decompress(data)
4267         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4268
4269         # Do a sanity check on various fields
4270         image = control.images['image']
4271         entries = image.GetEntries()
4272         self.assertEqual(2, len(entries))
4273
4274         entry = entries['blob']
4275         self.assertEqual(COMPRESS_DATA, entry.data)
4276         self.assertEqual(len(COMPRESS_DATA), entry.size)
4277
4278         entry = entries['u-boot']
4279         self.assertEqual(U_BOOT_DATA, entry.data)
4280         self.assertEqual(len(U_BOOT_DATA), entry.size)
4281
4282         self.assertEqual(len(data), image.size)
4283         self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, image.uncomp_data)
4284         self.assertEqual(len(COMPRESS_DATA + U_BOOT_DATA), image.uncomp_size)
4285         orig = self._decompress(image.data)
4286         self.assertEqual(orig, image.uncomp_data)
4287
4288         expected = {
4289             'blob:offset': 0,
4290             'blob:size': len(COMPRESS_DATA),
4291             'u-boot:offset': len(COMPRESS_DATA),
4292             'u-boot:size': len(U_BOOT_DATA),
4293             'uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4294             'offset': 0,
4295             'image-pos': 0,
4296             'size': len(data),
4297             }
4298         self.assertEqual(expected, props)
4299
4300     def testCompressImageLess(self):
4301         """Test compression where compression reduces the image size"""
4302         self._CheckLz4()
4303         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4304             '183_compress_image_less.dts', use_real_dtb=True, update_dtb=True)
4305         dtb = fdt.Fdt(out_dtb_fname)
4306         dtb.Scan()
4307         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4308                                         'uncomp-size'])
4309         orig = self._decompress(data)
4310
4311         self.assertEquals(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig)
4312
4313         # Do a sanity check on various fields
4314         image = control.images['image']
4315         entries = image.GetEntries()
4316         self.assertEqual(2, len(entries))
4317
4318         entry = entries['blob']
4319         self.assertEqual(COMPRESS_DATA_BIG, entry.data)
4320         self.assertEqual(len(COMPRESS_DATA_BIG), entry.size)
4321
4322         entry = entries['u-boot']
4323         self.assertEqual(U_BOOT_DATA, entry.data)
4324         self.assertEqual(len(U_BOOT_DATA), entry.size)
4325
4326         self.assertEqual(len(data), image.size)
4327         self.assertEqual(COMPRESS_DATA_BIG + U_BOOT_DATA, image.uncomp_data)
4328         self.assertEqual(len(COMPRESS_DATA_BIG + U_BOOT_DATA),
4329                          image.uncomp_size)
4330         orig = self._decompress(image.data)
4331         self.assertEqual(orig, image.uncomp_data)
4332
4333         expected = {
4334             'blob:offset': 0,
4335             'blob:size': len(COMPRESS_DATA_BIG),
4336             'u-boot:offset': len(COMPRESS_DATA_BIG),
4337             'u-boot:size': len(U_BOOT_DATA),
4338             'uncomp-size': len(COMPRESS_DATA_BIG + U_BOOT_DATA),
4339             'offset': 0,
4340             'image-pos': 0,
4341             'size': len(data),
4342             }
4343         self.assertEqual(expected, props)
4344
4345     def testCompressSectionSize(self):
4346         """Test compression of a section with a fixed size"""
4347         self._CheckLz4()
4348         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4349             '184_compress_section_size.dts', use_real_dtb=True, update_dtb=True)
4350         dtb = fdt.Fdt(out_dtb_fname)
4351         dtb.Scan()
4352         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4353                                         'uncomp-size'])
4354         orig = self._decompress(data)
4355         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4356         expected = {
4357             'section/blob:offset': 0,
4358             'section/blob:size': len(COMPRESS_DATA),
4359             'section/u-boot:offset': len(COMPRESS_DATA),
4360             'section/u-boot:size': len(U_BOOT_DATA),
4361             'section:offset': 0,
4362             'section:image-pos': 0,
4363             'section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4364             'section:size': 0x30,
4365             'offset': 0,
4366             'image-pos': 0,
4367             'size': 0x30,
4368             }
4369         self.assertEqual(expected, props)
4370
4371     def testCompressSection(self):
4372         """Test compression of a section with no fixed size"""
4373         self._CheckLz4()
4374         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4375             '185_compress_section.dts', use_real_dtb=True, update_dtb=True)
4376         dtb = fdt.Fdt(out_dtb_fname)
4377         dtb.Scan()
4378         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4379                                         'uncomp-size'])
4380         orig = self._decompress(data)
4381         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4382         expected = {
4383             'section/blob:offset': 0,
4384             'section/blob:size': len(COMPRESS_DATA),
4385             'section/u-boot:offset': len(COMPRESS_DATA),
4386             'section/u-boot:size': len(U_BOOT_DATA),
4387             'section:offset': 0,
4388             'section:image-pos': 0,
4389             'section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4390             'section:size': len(data),
4391             'offset': 0,
4392             'image-pos': 0,
4393             'size': len(data),
4394             }
4395         self.assertEqual(expected, props)
4396
4397     def testCompressExtra(self):
4398         """Test compression of a section with no fixed size"""
4399         self._CheckLz4()
4400         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4401             '186_compress_extra.dts', use_real_dtb=True, update_dtb=True)
4402         dtb = fdt.Fdt(out_dtb_fname)
4403         dtb.Scan()
4404         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4405                                         'uncomp-size'])
4406
4407         base = data[len(U_BOOT_DATA):]
4408         self.assertEquals(U_BOOT_DATA, base[:len(U_BOOT_DATA)])
4409         rest = base[len(U_BOOT_DATA):]
4410
4411         # Check compressed data
4412         section1 = self._decompress(rest)
4413         expect1 = comp_util.compress(COMPRESS_DATA + U_BOOT_DATA, 'lz4')
4414         self.assertEquals(expect1, rest[:len(expect1)])
4415         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1)
4416         rest1 = rest[len(expect1):]
4417
4418         section2 = self._decompress(rest1)
4419         expect2 = comp_util.compress(COMPRESS_DATA + COMPRESS_DATA, 'lz4')
4420         self.assertEquals(expect2, rest1[:len(expect2)])
4421         self.assertEquals(COMPRESS_DATA + COMPRESS_DATA, section2)
4422         rest2 = rest1[len(expect2):]
4423
4424         expect_size = (len(U_BOOT_DATA) + len(U_BOOT_DATA) + len(expect1) +
4425                        len(expect2) + len(U_BOOT_DATA))
4426         #self.assertEquals(expect_size, len(data))
4427
4428         #self.assertEquals(U_BOOT_DATA, rest2)
4429
4430         self.maxDiff = None
4431         expected = {
4432             'u-boot:offset': 0,
4433             'u-boot:image-pos': 0,
4434             'u-boot:size': len(U_BOOT_DATA),
4435
4436             'base:offset': len(U_BOOT_DATA),
4437             'base:image-pos': len(U_BOOT_DATA),
4438             'base:size': len(data) - len(U_BOOT_DATA),
4439             'base/u-boot:offset': 0,
4440             'base/u-boot:image-pos': len(U_BOOT_DATA),
4441             'base/u-boot:size': len(U_BOOT_DATA),
4442             'base/u-boot2:offset': len(U_BOOT_DATA) + len(expect1) +
4443                 len(expect2),
4444             'base/u-boot2:image-pos': len(U_BOOT_DATA) * 2 + len(expect1) +
4445                 len(expect2),
4446             'base/u-boot2:size': len(U_BOOT_DATA),
4447
4448             'base/section:offset': len(U_BOOT_DATA),
4449             'base/section:image-pos': len(U_BOOT_DATA) * 2,
4450             'base/section:size': len(expect1),
4451             'base/section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4452             'base/section/blob:offset': 0,
4453             'base/section/blob:size': len(COMPRESS_DATA),
4454             'base/section/u-boot:offset': len(COMPRESS_DATA),
4455             'base/section/u-boot:size': len(U_BOOT_DATA),
4456
4457             'base/section2:offset': len(U_BOOT_DATA) + len(expect1),
4458             'base/section2:image-pos': len(U_BOOT_DATA) * 2 + len(expect1),
4459             'base/section2:size': len(expect2),
4460             'base/section2:uncomp-size': len(COMPRESS_DATA + COMPRESS_DATA),
4461             'base/section2/blob:offset': 0,
4462             'base/section2/blob:size': len(COMPRESS_DATA),
4463             'base/section2/blob2:offset': len(COMPRESS_DATA),
4464             'base/section2/blob2:size': len(COMPRESS_DATA),
4465
4466             'offset': 0,
4467             'image-pos': 0,
4468             'size': len(data),
4469             }
4470         self.assertEqual(expected, props)
4471
4472     def testSymbolsSubsection(self):
4473         """Test binman can assign symbols from a subsection"""
4474         self.checkSymbols('187_symbols_sub.dts', U_BOOT_SPL_DATA, 0x18)
4475
4476     def testReadImageEntryArg(self):
4477         """Test reading an image that would need an entry arg to generate"""
4478         entry_args = {
4479             'cros-ec-rw-path': 'ecrw.bin',
4480         }
4481         data = self.data = self._DoReadFileDtb(
4482             '188_image_entryarg.dts',use_real_dtb=True, update_dtb=True,
4483             entry_args=entry_args)
4484
4485         image_fname = tools.get_output_filename('image.bin')
4486         orig_image = control.images['image']
4487
4488         # This should not generate an error about the missing 'cros-ec-rw-path'
4489         # since we are reading the image from a file. Compare with
4490         # testEntryArgsRequired()
4491         image = Image.FromFile(image_fname)
4492         self.assertEqual(orig_image.GetEntries().keys(),
4493                          image.GetEntries().keys())
4494
4495     def testFilesAlign(self):
4496         """Test alignment with files"""
4497         data = self._DoReadFile('190_files_align.dts')
4498
4499         # The first string is 15 bytes so will align to 16
4500         expect = FILES_DATA[:15] + b'\0' + FILES_DATA[15:]
4501         self.assertEqual(expect, data)
4502
4503     def testReadImageSkip(self):
4504         """Test reading an image and accessing its FDT map"""
4505         data = self.data = self._DoReadFileRealDtb('191_read_image_skip.dts')
4506         image_fname = tools.get_output_filename('image.bin')
4507         orig_image = control.images['image']
4508         image = Image.FromFile(image_fname)
4509         self.assertEqual(orig_image.GetEntries().keys(),
4510                          image.GetEntries().keys())
4511
4512         orig_entry = orig_image.GetEntries()['fdtmap']
4513         entry = image.GetEntries()['fdtmap']
4514         self.assertEqual(orig_entry.offset, entry.offset)
4515         self.assertEqual(orig_entry.size, entry.size)
4516         self.assertEqual(16, entry.image_pos)
4517
4518         u_boot = image.GetEntries()['section'].GetEntries()['u-boot']
4519
4520         self.assertEquals(U_BOOT_DATA, u_boot.ReadData())
4521
4522     def testTplNoDtb(self):
4523         """Test that an image with tpl/u-boot-tpl-nodtb.bin can be created"""
4524         self._SetupTplElf()
4525         data = self._DoReadFile('192_u_boot_tpl_nodtb.dts')
4526         self.assertEqual(U_BOOT_TPL_NODTB_DATA,
4527                          data[:len(U_BOOT_TPL_NODTB_DATA)])
4528
4529     def testTplBssPad(self):
4530         """Test that we can pad TPL's BSS with zeros"""
4531         # ELF file with a '__bss_size' symbol
4532         self._SetupTplElf()
4533         data = self._DoReadFile('193_tpl_bss_pad.dts')
4534         self.assertEqual(U_BOOT_TPL_DATA + tools.get_bytes(0, 10) + U_BOOT_DATA,
4535                          data)
4536
4537     def testTplBssPadMissing(self):
4538         """Test that a missing symbol is detected"""
4539         self._SetupTplElf('u_boot_ucode_ptr')
4540         with self.assertRaises(ValueError) as e:
4541             self._DoReadFile('193_tpl_bss_pad.dts')
4542         self.assertIn('Expected __bss_size symbol in tpl/u-boot-tpl',
4543                       str(e.exception))
4544
4545     def checkDtbSizes(self, data, pad_len, start):
4546         """Check the size arguments in a dtb embedded in an image
4547
4548         Args:
4549             data: The image data
4550             pad_len: Length of the pad section in the image, in bytes
4551             start: Start offset of the devicetree to examine, within the image
4552
4553         Returns:
4554             Size of the devicetree in bytes
4555         """
4556         dtb_data = data[start:]
4557         dtb = fdt.Fdt.FromData(dtb_data)
4558         fdt_size = dtb.GetFdtObj().totalsize()
4559         dtb.Scan()
4560         props = self._GetPropTree(dtb, 'size')
4561         self.assertEqual({
4562             'size': len(data),
4563             'u-boot-spl/u-boot-spl-bss-pad:size': pad_len,
4564             'u-boot-spl/u-boot-spl-dtb:size': 801,
4565             'u-boot-spl/u-boot-spl-nodtb:size': len(U_BOOT_SPL_NODTB_DATA),
4566             'u-boot-spl:size': 860,
4567             'u-boot-tpl:size': len(U_BOOT_TPL_DATA),
4568             'u-boot/u-boot-dtb:size': 781,
4569             'u-boot/u-boot-nodtb:size': len(U_BOOT_NODTB_DATA),
4570             'u-boot:size': 827,
4571             }, props)
4572         return fdt_size
4573
4574     def testExpanded(self):
4575         """Test that an expanded entry type is selected when needed"""
4576         self._SetupSplElf()
4577         self._SetupTplElf()
4578
4579         # SPL has a devicetree, TPL does not
4580         entry_args = {
4581             'spl-dtb': '1',
4582             'spl-bss-pad': 'y',
4583             'tpl-dtb': '',
4584         }
4585         self._DoReadFileDtb('194_fdt_incl.dts', use_expanded=True,
4586                             entry_args=entry_args)
4587         image = control.images['image']
4588         entries = image.GetEntries()
4589         self.assertEqual(3, len(entries))
4590
4591         # First, u-boot, which should be expanded into u-boot-nodtb and dtb
4592         self.assertIn('u-boot', entries)
4593         entry = entries['u-boot']
4594         self.assertEqual('u-boot-expanded', entry.etype)
4595         subent = entry.GetEntries()
4596         self.assertEqual(2, len(subent))
4597         self.assertIn('u-boot-nodtb', subent)
4598         self.assertIn('u-boot-dtb', subent)
4599
4600         # Second, u-boot-spl, which should be expanded into three parts
4601         self.assertIn('u-boot-spl', entries)
4602         entry = entries['u-boot-spl']
4603         self.assertEqual('u-boot-spl-expanded', entry.etype)
4604         subent = entry.GetEntries()
4605         self.assertEqual(3, len(subent))
4606         self.assertIn('u-boot-spl-nodtb', subent)
4607         self.assertIn('u-boot-spl-bss-pad', subent)
4608         self.assertIn('u-boot-spl-dtb', subent)
4609
4610         # Third, u-boot-tpl, which should be not be expanded, since TPL has no
4611         # devicetree
4612         self.assertIn('u-boot-tpl', entries)
4613         entry = entries['u-boot-tpl']
4614         self.assertEqual('u-boot-tpl', entry.etype)
4615         self.assertEqual(None, entry.GetEntries())
4616
4617     def testExpandedTpl(self):
4618         """Test that an expanded entry type is selected for TPL when needed"""
4619         self._SetupTplElf()
4620
4621         entry_args = {
4622             'tpl-bss-pad': 'y',
4623             'tpl-dtb': 'y',
4624         }
4625         self._DoReadFileDtb('195_fdt_incl_tpl.dts', use_expanded=True,
4626                             entry_args=entry_args)
4627         image = control.images['image']
4628         entries = image.GetEntries()
4629         self.assertEqual(1, len(entries))
4630
4631         # We only have u-boot-tpl, which be expanded
4632         self.assertIn('u-boot-tpl', entries)
4633         entry = entries['u-boot-tpl']
4634         self.assertEqual('u-boot-tpl-expanded', entry.etype)
4635         subent = entry.GetEntries()
4636         self.assertEqual(3, len(subent))
4637         self.assertIn('u-boot-tpl-nodtb', subent)
4638         self.assertIn('u-boot-tpl-bss-pad', subent)
4639         self.assertIn('u-boot-tpl-dtb', subent)
4640
4641     def testExpandedNoPad(self):
4642         """Test an expanded entry without BSS pad enabled"""
4643         self._SetupSplElf()
4644         self._SetupTplElf()
4645
4646         # SPL has a devicetree, TPL does not
4647         entry_args = {
4648             'spl-dtb': 'something',
4649             'spl-bss-pad': 'n',
4650             'tpl-dtb': '',
4651         }
4652         self._DoReadFileDtb('194_fdt_incl.dts', use_expanded=True,
4653                             entry_args=entry_args)
4654         image = control.images['image']
4655         entries = image.GetEntries()
4656
4657         # Just check u-boot-spl, which should be expanded into two parts
4658         self.assertIn('u-boot-spl', entries)
4659         entry = entries['u-boot-spl']
4660         self.assertEqual('u-boot-spl-expanded', entry.etype)
4661         subent = entry.GetEntries()
4662         self.assertEqual(2, len(subent))
4663         self.assertIn('u-boot-spl-nodtb', subent)
4664         self.assertIn('u-boot-spl-dtb', subent)
4665
4666     def testExpandedTplNoPad(self):
4667         """Test that an expanded entry type with padding disabled in TPL"""
4668         self._SetupTplElf()
4669
4670         entry_args = {
4671             'tpl-bss-pad': '',
4672             'tpl-dtb': 'y',
4673         }
4674         self._DoReadFileDtb('195_fdt_incl_tpl.dts', use_expanded=True,
4675                             entry_args=entry_args)
4676         image = control.images['image']
4677         entries = image.GetEntries()
4678         self.assertEqual(1, len(entries))
4679
4680         # We only have u-boot-tpl, which be expanded
4681         self.assertIn('u-boot-tpl', entries)
4682         entry = entries['u-boot-tpl']
4683         self.assertEqual('u-boot-tpl-expanded', entry.etype)
4684         subent = entry.GetEntries()
4685         self.assertEqual(2, len(subent))
4686         self.assertIn('u-boot-tpl-nodtb', subent)
4687         self.assertIn('u-boot-tpl-dtb', subent)
4688
4689     def testFdtInclude(self):
4690         """Test that an Fdt is update within all binaries"""
4691         self._SetupSplElf()
4692         self._SetupTplElf()
4693
4694         # SPL has a devicetree, TPL does not
4695         self.maxDiff = None
4696         entry_args = {
4697             'spl-dtb': '1',
4698             'spl-bss-pad': 'y',
4699             'tpl-dtb': '',
4700         }
4701         # Build the image. It includes two separate devicetree binaries, each
4702         # with their own contents, but all contain the binman definition.
4703         data = self._DoReadFileDtb(
4704             '194_fdt_incl.dts', use_real_dtb=True, use_expanded=True,
4705             update_dtb=True, entry_args=entry_args)[0]
4706         pad_len = 10
4707
4708         # Check the U-Boot dtb
4709         start = len(U_BOOT_NODTB_DATA)
4710         fdt_size = self.checkDtbSizes(data, pad_len, start)
4711
4712         # Now check SPL
4713         start += fdt_size + len(U_BOOT_SPL_NODTB_DATA) + pad_len
4714         fdt_size = self.checkDtbSizes(data, pad_len, start)
4715
4716         # TPL has no devicetree
4717         start += fdt_size + len(U_BOOT_TPL_DATA)
4718         self.assertEqual(len(data), start)
4719
4720     def testSymbolsExpanded(self):
4721         """Test binman can assign symbols in expanded entries"""
4722         entry_args = {
4723             'spl-dtb': '1',
4724         }
4725         self.checkSymbols('197_symbols_expand.dts', U_BOOT_SPL_NODTB_DATA +
4726                           U_BOOT_SPL_DTB_DATA, 0x38,
4727                           entry_args=entry_args, use_expanded=True)
4728
4729     def testCollection(self):
4730         """Test a collection"""
4731         data = self._DoReadFile('198_collection.dts')
4732         self.assertEqual(U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA +
4733                          tools.get_bytes(0xff, 2) + U_BOOT_NODTB_DATA +
4734                          tools.get_bytes(0xfe, 3) + U_BOOT_DTB_DATA,
4735                          data)
4736
4737     def testCollectionSection(self):
4738         """Test a collection where a section must be built first"""
4739         # Sections never have their contents when GetData() is called, but when
4740         # BuildSectionData() is called with required=True, a section will force
4741         # building the contents, producing an error is anything is still
4742         # missing.
4743         data = self._DoReadFile('199_collection_section.dts')
4744         section = U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA
4745         self.assertEqual(section + U_BOOT_DATA + tools.get_bytes(0xff, 2) +
4746                          section + tools.get_bytes(0xfe, 3) + U_BOOT_DATA,
4747                          data)
4748
4749     def testAlignDefault(self):
4750         """Test that default alignment works on sections"""
4751         data = self._DoReadFile('200_align_default.dts')
4752         expected = (U_BOOT_DATA + tools.get_bytes(0, 8 - len(U_BOOT_DATA)) +
4753                     U_BOOT_DATA)
4754         # Special alignment for section
4755         expected += tools.get_bytes(0, 32 - len(expected))
4756         # No alignment within the nested section
4757         expected += U_BOOT_DATA + U_BOOT_NODTB_DATA;
4758         # Now the final piece, which should be default-aligned
4759         expected += tools.get_bytes(0, 88 - len(expected)) + U_BOOT_NODTB_DATA
4760         self.assertEqual(expected, data)
4761
4762     def testPackOpenSBI(self):
4763         """Test that an image with an OpenSBI binary can be created"""
4764         data = self._DoReadFile('201_opensbi.dts')
4765         self.assertEqual(OPENSBI_DATA, data[:len(OPENSBI_DATA)])
4766
4767     def testSectionsSingleThread(self):
4768         """Test sections without multithreading"""
4769         data = self._DoReadFileDtb('055_sections.dts', threads=0)[0]
4770         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
4771                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12) +
4772                     U_BOOT_DATA + tools.get_bytes(ord('&'), 4))
4773         self.assertEqual(expected, data)
4774
4775     def testThreadTimeout(self):
4776         """Test handling a thread that takes too long"""
4777         with self.assertRaises(ValueError) as e:
4778             self._DoTestFile('202_section_timeout.dts',
4779                              test_section_timeout=True)
4780         self.assertIn("Timed out obtaining contents", str(e.exception))
4781
4782     def testTiming(self):
4783         """Test output of timing information"""
4784         data = self._DoReadFile('055_sections.dts')
4785         with test_util.capture_sys_output() as (stdout, stderr):
4786             state.TimingShow()
4787         self.assertIn('read:', stdout.getvalue())
4788         self.assertIn('compress:', stdout.getvalue())
4789
4790     def testUpdateFdtInElf(self):
4791         """Test that we can update the devicetree in an ELF file"""
4792         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed')
4793         outfile = os.path.join(self._indir, 'u-boot.out')
4794         begin_sym = 'dtb_embed_begin'
4795         end_sym = 'dtb_embed_end'
4796         retcode = self._DoTestFile(
4797             '060_fdt_update.dts', update_dtb=True,
4798             update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4799         self.assertEqual(0, retcode)
4800
4801         # Check that the output file does in fact contact a dtb with the binman
4802         # definition in the correct place
4803         syms = elf.GetSymbolFileOffset(infile,
4804                                        ['dtb_embed_begin', 'dtb_embed_end'])
4805         data = tools.read_file(outfile)
4806         dtb_data = data[syms['dtb_embed_begin'].offset:
4807                         syms['dtb_embed_end'].offset]
4808
4809         dtb = fdt.Fdt.FromData(dtb_data)
4810         dtb.Scan()
4811         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
4812         self.assertEqual({
4813             'image-pos': 0,
4814             'offset': 0,
4815             '_testing:offset': 32,
4816             '_testing:size': 2,
4817             '_testing:image-pos': 32,
4818             'section@0/u-boot:offset': 0,
4819             'section@0/u-boot:size': len(U_BOOT_DATA),
4820             'section@0/u-boot:image-pos': 0,
4821             'section@0:offset': 0,
4822             'section@0:size': 16,
4823             'section@0:image-pos': 0,
4824
4825             'section@1/u-boot:offset': 0,
4826             'section@1/u-boot:size': len(U_BOOT_DATA),
4827             'section@1/u-boot:image-pos': 16,
4828             'section@1:offset': 16,
4829             'section@1:size': 16,
4830             'section@1:image-pos': 16,
4831             'size': 40
4832         }, props)
4833
4834     def testUpdateFdtInElfInvalid(self):
4835         """Test that invalid args are detected with --update-fdt-in-elf"""
4836         with self.assertRaises(ValueError) as e:
4837             self._DoTestFile('060_fdt_update.dts', update_fdt_in_elf='fred')
4838         self.assertIn("Invalid args ['fred'] to --update-fdt-in-elf",
4839                       str(e.exception))
4840
4841     def testUpdateFdtInElfNoSyms(self):
4842         """Test that missing symbols are detected with --update-fdt-in-elf"""
4843         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed')
4844         outfile = ''
4845         begin_sym = 'wrong_begin'
4846         end_sym = 'wrong_end'
4847         with self.assertRaises(ValueError) as e:
4848             self._DoTestFile(
4849                 '060_fdt_update.dts',
4850                 update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4851         self.assertIn("Expected two symbols 'wrong_begin' and 'wrong_end': got 0:",
4852                       str(e.exception))
4853
4854     def testUpdateFdtInElfTooSmall(self):
4855         """Test that an over-large dtb is detected with --update-fdt-in-elf"""
4856         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed_sm')
4857         outfile = os.path.join(self._indir, 'u-boot.out')
4858         begin_sym = 'dtb_embed_begin'
4859         end_sym = 'dtb_embed_end'
4860         with self.assertRaises(ValueError) as e:
4861             self._DoTestFile(
4862                 '060_fdt_update.dts', update_dtb=True,
4863                 update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4864         self.assertRegex(
4865             str(e.exception),
4866             "Not enough space in '.*u_boot_binman_embed_sm' for data length.*")
4867
4868     def testVersion(self):
4869         """Test we can get the binman version"""
4870         version = '(unreleased)'
4871         self.assertEqual(version, state.GetVersion(self._indir))
4872
4873         with self.assertRaises(SystemExit):
4874             with test_util.capture_sys_output() as (_, stderr):
4875                 self._DoBinman('-V')
4876         self.assertEqual('Binman %s\n' % version, stderr.getvalue())
4877
4878         # Try running the tool too, just to be safe
4879         result = self._RunBinman('-V')
4880         self.assertEqual('Binman %s\n' % version, result.stderr)
4881
4882         # Set up a version file to make sure that works
4883         version = 'v2025.01-rc2'
4884         tools.write_file(os.path.join(self._indir, 'version'), version,
4885                         binary=False)
4886         self.assertEqual(version, state.GetVersion(self._indir))
4887
4888     def testAltFormat(self):
4889         """Test that alternative formats can be used to extract"""
4890         self._DoReadFileRealDtb('213_fdtmap_alt_format.dts')
4891
4892         try:
4893             tmpdir, updated_fname = self._SetupImageInTmpdir()
4894             with test_util.capture_sys_output() as (stdout, _):
4895                 self._DoBinman('extract', '-i', updated_fname, '-F', 'list')
4896             self.assertEqual(
4897                 '''Flag (-F)   Entry type            Description
4898 fdt         fdtmap                Extract the devicetree blob from the fdtmap
4899 ''',
4900                 stdout.getvalue())
4901
4902             dtb = os.path.join(tmpdir, 'fdt.dtb')
4903             self._DoBinman('extract', '-i', updated_fname, '-F', 'fdt', '-f',
4904                            dtb, 'fdtmap')
4905
4906             # Check that we can read it and it can be scanning, meaning it does
4907             # not have a 16-byte fdtmap header
4908             data = tools.read_file(dtb)
4909             dtb = fdt.Fdt.FromData(data)
4910             dtb.Scan()
4911
4912             # Now check u-boot which has no alt_format
4913             fname = os.path.join(tmpdir, 'fdt.dtb')
4914             self._DoBinman('extract', '-i', updated_fname, '-F', 'dummy',
4915                            '-f', fname, 'u-boot')
4916             data = tools.read_file(fname)
4917             self.assertEqual(U_BOOT_DATA, data)
4918
4919         finally:
4920             shutil.rmtree(tmpdir)
4921
4922     def testExtblobList(self):
4923         """Test an image with an external blob list"""
4924         data = self._DoReadFile('215_blob_ext_list.dts')
4925         self.assertEqual(REFCODE_DATA + FSP_M_DATA, data)
4926
4927     def testExtblobListMissing(self):
4928         """Test an image with a missing external blob"""
4929         with self.assertRaises(ValueError) as e:
4930             self._DoReadFile('216_blob_ext_list_missing.dts')
4931         self.assertIn("Filename 'missing-file' not found in input path",
4932                       str(e.exception))
4933
4934     def testExtblobListMissingOk(self):
4935         """Test an image with an missing external blob that is allowed"""
4936         with test_util.capture_sys_output() as (stdout, stderr):
4937             self._DoTestFile('216_blob_ext_list_missing.dts',
4938                              allow_missing=True)
4939         err = stderr.getvalue()
4940         self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext")
4941
4942     def testFip(self):
4943         """Basic test of generation of an ARM Firmware Image Package (FIP)"""
4944         data = self._DoReadFile('203_fip.dts')
4945         hdr, fents = fip_util.decode_fip(data)
4946         self.assertEqual(fip_util.HEADER_MAGIC, hdr.name)
4947         self.assertEqual(fip_util.HEADER_SERIAL, hdr.serial)
4948         self.assertEqual(0x123, hdr.flags)
4949
4950         self.assertEqual(2, len(fents))
4951
4952         fent = fents[0]
4953         self.assertEqual(
4954             bytes([0x47,  0xd4, 0x08, 0x6d, 0x4c, 0xfe, 0x98, 0x46,
4955                   0x9b, 0x95, 0x29, 0x50, 0xcb, 0xbd, 0x5a, 0x0]), fent.uuid)
4956         self.assertEqual('soc-fw', fent.fip_type)
4957         self.assertEqual(0x88, fent.offset)
4958         self.assertEqual(len(ATF_BL31_DATA), fent.size)
4959         self.assertEqual(0x123456789abcdef, fent.flags)
4960         self.assertEqual(ATF_BL31_DATA, fent.data)
4961         self.assertEqual(True, fent.valid)
4962
4963         fent = fents[1]
4964         self.assertEqual(
4965             bytes([0x65, 0x92, 0x27, 0x03, 0x2f, 0x74, 0xe6, 0x44,
4966              0x8d, 0xff, 0x57, 0x9a, 0xc1, 0xff, 0x06, 0x10]), fent.uuid)
4967         self.assertEqual('scp-fwu-cfg', fent.fip_type)
4968         self.assertEqual(0x8c, fent.offset)
4969         self.assertEqual(len(ATF_BL31_DATA), fent.size)
4970         self.assertEqual(0, fent.flags)
4971         self.assertEqual(ATF_BL2U_DATA, fent.data)
4972         self.assertEqual(True, fent.valid)
4973
4974     def testFipOther(self):
4975         """Basic FIP with something that isn't a external blob"""
4976         data = self._DoReadFile('204_fip_other.dts')
4977         hdr, fents = fip_util.decode_fip(data)
4978
4979         self.assertEqual(2, len(fents))
4980         fent = fents[1]
4981         self.assertEqual('rot-cert', fent.fip_type)
4982         self.assertEqual(b'aa', fent.data)
4983
4984     def testFipNoType(self):
4985         """FIP with an entry of an unknown type"""
4986         with self.assertRaises(ValueError) as e:
4987             self._DoReadFile('205_fip_no_type.dts')
4988         self.assertIn("Must provide a fip-type (node name 'u-boot' is not a known FIP type)",
4989                       str(e.exception))
4990
4991     def testFipUuid(self):
4992         """Basic FIP with a manual uuid"""
4993         data = self._DoReadFile('206_fip_uuid.dts')
4994         hdr, fents = fip_util.decode_fip(data)
4995
4996         self.assertEqual(2, len(fents))
4997         fent = fents[1]
4998         self.assertEqual(None, fent.fip_type)
4999         self.assertEqual(
5000             bytes([0xfc, 0x65, 0x13, 0x92, 0x4a, 0x5b, 0x11, 0xec,
5001                    0x94, 0x35, 0xff, 0x2d, 0x1c, 0xfc, 0x79, 0x9c]),
5002             fent.uuid)
5003         self.assertEqual(U_BOOT_DATA, fent.data)
5004
5005     def testFipLs(self):
5006         """Test listing a FIP"""
5007         data = self._DoReadFileRealDtb('207_fip_ls.dts')
5008         hdr, fents = fip_util.decode_fip(data)
5009
5010         try:
5011             tmpdir, updated_fname = self._SetupImageInTmpdir()
5012             with test_util.capture_sys_output() as (stdout, stderr):
5013                 self._DoBinman('ls', '-i', updated_fname)
5014         finally:
5015             shutil.rmtree(tmpdir)
5016         lines = stdout.getvalue().splitlines()
5017         expected = [
5018 'Name          Image-pos  Size  Entry-type  Offset  Uncomp-size',
5019 '----------------------------------------------------------------',
5020 'main-section          0   2d3  section          0',
5021 '  atf-fip             0    90  atf-fip          0',
5022 '    soc-fw           88     4  blob-ext        88',
5023 '    u-boot           8c     4  u-boot          8c',
5024 '  fdtmap             90   243  fdtmap          90',
5025 ]
5026         self.assertEqual(expected, lines)
5027
5028         image = control.images['image']
5029         entries = image.GetEntries()
5030         fdtmap = entries['fdtmap']
5031
5032         fdtmap_data = data[fdtmap.image_pos:fdtmap.image_pos + fdtmap.size]
5033         magic = fdtmap_data[:8]
5034         self.assertEqual(b'_FDTMAP_', magic)
5035         self.assertEqual(tools.get_bytes(0, 8), fdtmap_data[8:16])
5036
5037         fdt_data = fdtmap_data[16:]
5038         dtb = fdt.Fdt.FromData(fdt_data)
5039         dtb.Scan()
5040         props = self._GetPropTree(dtb, BASE_DTB_PROPS, prefix='/')
5041         self.assertEqual({
5042             'atf-fip/soc-fw:image-pos': 136,
5043             'atf-fip/soc-fw:offset': 136,
5044             'atf-fip/soc-fw:size': 4,
5045             'atf-fip/u-boot:image-pos': 140,
5046             'atf-fip/u-boot:offset': 140,
5047             'atf-fip/u-boot:size': 4,
5048             'atf-fip:image-pos': 0,
5049             'atf-fip:offset': 0,
5050             'atf-fip:size': 144,
5051             'image-pos': 0,
5052             'offset': 0,
5053             'fdtmap:image-pos': fdtmap.image_pos,
5054             'fdtmap:offset': fdtmap.offset,
5055             'fdtmap:size': len(fdtmap_data),
5056             'size': len(data),
5057         }, props)
5058
5059     def testFipExtractOneEntry(self):
5060         """Test extracting a single entry fron an FIP"""
5061         self._DoReadFileRealDtb('207_fip_ls.dts')
5062         image_fname = tools.get_output_filename('image.bin')
5063         fname = os.path.join(self._indir, 'output.extact')
5064         control.ExtractEntries(image_fname, fname, None, ['atf-fip/u-boot'])
5065         data = tools.read_file(fname)
5066         self.assertEqual(U_BOOT_DATA, data)
5067
5068     def testFipReplace(self):
5069         """Test replacing a single file in a FIP"""
5070         expected = U_BOOT_DATA + tools.get_bytes(0x78, 50)
5071         data = self._DoReadFileRealDtb('208_fip_replace.dts')
5072         updated_fname = tools.get_output_filename('image-updated.bin')
5073         tools.write_file(updated_fname, data)
5074         entry_name = 'atf-fip/u-boot'
5075         control.WriteEntry(updated_fname, entry_name, expected,
5076                            allow_resize=True)
5077         actual = control.ReadEntry(updated_fname, entry_name)
5078         self.assertEqual(expected, actual)
5079
5080         new_data = tools.read_file(updated_fname)
5081         hdr, fents = fip_util.decode_fip(new_data)
5082
5083         self.assertEqual(2, len(fents))
5084
5085         # Check that the FIP entry is updated
5086         fent = fents[1]
5087         self.assertEqual(0x8c, fent.offset)
5088         self.assertEqual(len(expected), fent.size)
5089         self.assertEqual(0, fent.flags)
5090         self.assertEqual(expected, fent.data)
5091         self.assertEqual(True, fent.valid)
5092
5093     def testFipMissing(self):
5094         with test_util.capture_sys_output() as (stdout, stderr):
5095             self._DoTestFile('209_fip_missing.dts', allow_missing=True)
5096         err = stderr.getvalue()
5097         self.assertRegex(err, "Image 'main-section'.*missing.*: rmm-fw")
5098
5099     def testFipSize(self):
5100         """Test a FIP with a size property"""
5101         data = self._DoReadFile('210_fip_size.dts')
5102         self.assertEqual(0x100 + len(U_BOOT_DATA), len(data))
5103         hdr, fents = fip_util.decode_fip(data)
5104         self.assertEqual(fip_util.HEADER_MAGIC, hdr.name)
5105         self.assertEqual(fip_util.HEADER_SERIAL, hdr.serial)
5106
5107         self.assertEqual(1, len(fents))
5108
5109         fent = fents[0]
5110         self.assertEqual('soc-fw', fent.fip_type)
5111         self.assertEqual(0x60, fent.offset)
5112         self.assertEqual(len(ATF_BL31_DATA), fent.size)
5113         self.assertEqual(ATF_BL31_DATA, fent.data)
5114         self.assertEqual(True, fent.valid)
5115
5116         rest = data[0x60 + len(ATF_BL31_DATA):0x100]
5117         self.assertEqual(tools.get_bytes(0xff, len(rest)), rest)
5118
5119     def testFipBadAlign(self):
5120         """Test that an invalid alignment value in a FIP is detected"""
5121         with self.assertRaises(ValueError) as e:
5122             self._DoTestFile('211_fip_bad_align.dts')
5123         self.assertIn(
5124             "Node \'/binman/atf-fip\': FIP alignment 31 must be a power of two",
5125             str(e.exception))
5126
5127     def testFipCollection(self):
5128         """Test using a FIP in a collection"""
5129         data = self._DoReadFile('212_fip_collection.dts')
5130         entry1 = control.images['image'].GetEntries()['collection']
5131         data1 = data[:entry1.size]
5132         hdr1, fents2 = fip_util.decode_fip(data1)
5133
5134         entry2 = control.images['image'].GetEntries()['atf-fip']
5135         data2 = data[entry2.offset:entry2.offset + entry2.size]
5136         hdr1, fents2 = fip_util.decode_fip(data2)
5137
5138         # The 'collection' entry should have U-Boot included at the end
5139         self.assertEqual(entry1.size - len(U_BOOT_DATA), entry2.size)
5140         self.assertEqual(data1, data2 + U_BOOT_DATA)
5141         self.assertEqual(U_BOOT_DATA, data1[-4:])
5142
5143         # There should be a U-Boot after the final FIP
5144         self.assertEqual(U_BOOT_DATA, data[-4:])
5145
5146     def testFakeBlob(self):
5147         """Test handling of faking an external blob"""
5148         with test_util.capture_sys_output() as (stdout, stderr):
5149             self._DoTestFile('217_fake_blob.dts', allow_missing=True,
5150                              allow_fake_blobs=True)
5151         err = stderr.getvalue()
5152         self.assertRegex(
5153             err,
5154             "Image '.*' has faked external blobs and is non-functional: .*")
5155
5156     def testExtblobListFaked(self):
5157         """Test an extblob with missing external blob that are faked"""
5158         with test_util.capture_sys_output() as (stdout, stderr):
5159             self._DoTestFile('216_blob_ext_list_missing.dts',
5160                              allow_fake_blobs=True)
5161         err = stderr.getvalue()
5162         self.assertRegex(err, "Image 'main-section'.*faked.*: blob-ext-list")
5163
5164     def testListBintools(self):
5165         args = ['tool', '--list']
5166         with test_util.capture_sys_output() as (stdout, _):
5167             self._DoBinman(*args)
5168         out = stdout.getvalue().splitlines()
5169         self.assertTrue(len(out) >= 2)
5170
5171     def testFetchBintools(self):
5172         def fail_download(url):
5173             """Take the tools.download() function by raising an exception"""
5174             raise urllib.error.URLError('my error')
5175
5176         args = ['tool']
5177         with self.assertRaises(ValueError) as e:
5178             self._DoBinman(*args)
5179         self.assertIn("Invalid arguments to 'tool' subcommand",
5180                       str(e.exception))
5181
5182         args = ['tool', '--fetch']
5183         with self.assertRaises(ValueError) as e:
5184             self._DoBinman(*args)
5185         self.assertIn('Please specify bintools to fetch', str(e.exception))
5186
5187         args = ['tool', '--fetch', '_testing']
5188         with unittest.mock.patch.object(tools, 'download',
5189                                         side_effect=fail_download):
5190             with test_util.capture_sys_output() as (stdout, _):
5191                 self._DoBinman(*args)
5192         self.assertIn('failed to fetch with all methods', stdout.getvalue())
5193
5194     def testInvalidCompress(self):
5195         with self.assertRaises(ValueError) as e:
5196             comp_util.compress(b'', 'invalid')
5197         self.assertIn("Unknown algorithm 'invalid'", str(e.exception))
5198
5199         with self.assertRaises(ValueError) as e:
5200             comp_util.decompress(b'1234', 'invalid')
5201         self.assertIn("Unknown algorithm 'invalid'", str(e.exception))
5202
5203     def testBintoolDocs(self):
5204         """Test for creation of bintool documentation"""
5205         with test_util.capture_sys_output() as (stdout, stderr):
5206             control.write_bintool_docs(control.bintool.Bintool.get_tool_list())
5207         self.assertTrue(len(stdout.getvalue()) > 0)
5208
5209     def testBintoolDocsMissing(self):
5210         """Test handling of missing bintool documentation"""
5211         with self.assertRaises(ValueError) as e:
5212             with test_util.capture_sys_output() as (stdout, stderr):
5213                 control.write_bintool_docs(
5214                     control.bintool.Bintool.get_tool_list(), 'mkimage')
5215         self.assertIn('Documentation is missing for modules: mkimage',
5216                       str(e.exception))
5217
5218     def testListWithGenNode(self):
5219         """Check handling of an FDT map when the section cannot be found"""
5220         entry_args = {
5221             'of-list': 'test-fdt1 test-fdt2',
5222         }
5223         data = self._DoReadFileDtb(
5224             '219_fit_gennode.dts',
5225             entry_args=entry_args,
5226             use_real_dtb=True,
5227             extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])
5228
5229         try:
5230             tmpdir, updated_fname = self._SetupImageInTmpdir()
5231             with test_util.capture_sys_output() as (stdout, stderr):
5232                 self._RunBinman('ls', '-i', updated_fname)
5233         finally:
5234             shutil.rmtree(tmpdir)
5235
5236     def testFitSubentryUsesBintool(self):
5237         """Test that binman FIT subentries can use bintools"""
5238         command.test_result = self._HandleGbbCommand
5239         entry_args = {
5240             'keydir': 'devkeys',
5241             'bmpblk': 'bmpblk.bin',
5242         }
5243         data, _, _, _ = self._DoReadFileDtb('220_fit_subentry_bintool.dts',
5244                 entry_args=entry_args)
5245
5246         expected = (GBB_DATA + GBB_DATA + tools.get_bytes(0, 8) +
5247                     tools.get_bytes(0, 0x2180 - 16))
5248         self.assertIn(expected, data)
5249
5250     def testFitSubentryMissingBintool(self):
5251         """Test that binman reports missing bintools for FIT subentries"""
5252         entry_args = {
5253             'keydir': 'devkeys',
5254         }
5255         with test_util.capture_sys_output() as (_, stderr):
5256             self._DoTestFile('220_fit_subentry_bintool.dts',
5257                     force_missing_bintools='futility', entry_args=entry_args)
5258         err = stderr.getvalue()
5259         self.assertRegex(err,
5260                          "Image 'main-section'.*missing bintools.*: futility")
5261
5262     def testFitSubentryHashSubnode(self):
5263         """Test an image with a FIT inside"""
5264         data, _, _, out_dtb_name = self._DoReadFileDtb(
5265             '221_fit_subentry_hash.dts', use_real_dtb=True, update_dtb=True)
5266
5267         mkimage_dtb = fdt.Fdt.FromData(data)
5268         mkimage_dtb.Scan()
5269         binman_dtb = fdt.Fdt(out_dtb_name)
5270         binman_dtb.Scan()
5271
5272         # Check that binman didn't add hash values
5273         fnode = binman_dtb.GetNode('/binman/fit/images/kernel/hash')
5274         self.assertNotIn('value', fnode.props)
5275
5276         fnode = binman_dtb.GetNode('/binman/fit/images/fdt-1/hash')
5277         self.assertNotIn('value', fnode.props)
5278
5279         # Check that mkimage added hash values
5280         fnode = mkimage_dtb.GetNode('/images/kernel/hash')
5281         self.assertIn('value', fnode.props)
5282
5283         fnode = mkimage_dtb.GetNode('/images/fdt-1/hash')
5284         self.assertIn('value', fnode.props)
5285
5286     def testPackTeeOs(self):
5287         """Test that an image with an TEE binary can be created"""
5288         data = self._DoReadFile('222_tee_os.dts')
5289         self.assertEqual(TEE_OS_DATA, data[:len(TEE_OS_DATA)])
5290
5291     def testFitFdtOper(self):
5292         """Check handling of a specified FIT operation"""
5293         entry_args = {
5294             'of-list': 'test-fdt1 test-fdt2',
5295             'default-dt': 'test-fdt2',
5296         }
5297         self._DoReadFileDtb(
5298             '223_fit_fdt_oper.dts',
5299             entry_args=entry_args,
5300             extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
5301
5302     def testFitFdtBadOper(self):
5303         """Check handling of an FDT map when the section cannot be found"""
5304         with self.assertRaises(ValueError) as exc:
5305             self._DoReadFileDtb('224_fit_bad_oper.dts')
5306         self.assertIn("Node '/binman/fit': Unknown operation 'unknown'",
5307                       str(exc.exception))
5308
5309     def test_uses_expand_size(self):
5310         """Test that the 'expand-size' property cannot be used anymore"""
5311         with self.assertRaises(ValueError) as e:
5312            data = self._DoReadFile('225_expand_size_bad.dts')
5313         self.assertIn(
5314             "Node '/binman/u-boot': Please use 'extend-size' instead of 'expand-size'",
5315             str(e.exception))
5316
5317     def testMkimageMissingBlob(self):
5318         """Test using mkimage to build an image"""
5319         with test_util.capture_sys_output() as (stdout, stderr):
5320             self._DoTestFile('229_mkimage_missing.dts', allow_missing=True,
5321                              allow_fake_blobs=True)
5322         err = stderr.getvalue()
5323         self.assertRegex(
5324             err,
5325             "Image '.*' has faked external blobs and is non-functional: .*")
5326
5327
5328 if __name__ == "__main__":
5329     unittest.main()