tools/binman/ftest.py

   1 # SPDX-License-Identifier: GPL-2.0+
   2 # Copyright (c) 2016 Google, Inc
   3 # Written by Simon Glass <sjg@chromium.org>
   4 #
   5 # To run a single test, change to this directory, and:
   6 #
   7 #    python -m unittest func_test.TestFunctional.testHelp
   8
   9 import collections
  10 import gzip
  11 import hashlib
  12 from optparse import OptionParser
  13 import os
  14 import re
  15 import shutil
  16 import struct
  17 import sys
  18 import tempfile
  19 import unittest
  20 import unittest.mock
  21 import urllib.error
  22
  23 from binman import bintool
  24 from binman import cbfs_util
  25 from binman import cmdline
  26 from binman import comp_util
  27 from binman import control
  28 from binman import elf
  29 from binman import elf_test
  30 from binman import fip_util
  31 from binman import fmap_util
  32 from binman import state
  33 from dtoc import fdt
  34 from dtoc import fdt_util
  35 from binman.etype import fdtmap
  36 from binman.etype import image_header
  37 from binman.image import Image
  38 from patman import command
  39 from patman import test_util
  40 from patman import tools
  41 from patman import tout
  42
  43 # Contents of test files, corresponding to different entry types
  44 U_BOOT_DATA           = b'1234'
  45 U_BOOT_IMG_DATA       = b'img'
  46 U_BOOT_SPL_DATA       = b'56780123456789abcdefghi'
  47 U_BOOT_TPL_DATA       = b'tpl9876543210fedcbazyw'
  48 BLOB_DATA             = b'89'
  49 ME_DATA               = b'0abcd'
  50 VGA_DATA              = b'vga'
  51 U_BOOT_DTB_DATA       = b'udtb'
  52 U_BOOT_SPL_DTB_DATA   = b'spldtb'
  53 U_BOOT_TPL_DTB_DATA   = b'tpldtb'
  54 X86_START16_DATA      = b'start16'
  55 X86_START16_SPL_DATA  = b'start16spl'
  56 X86_START16_TPL_DATA  = b'start16tpl'
  57 X86_RESET16_DATA      = b'reset16'
  58 X86_RESET16_SPL_DATA  = b'reset16spl'
  59 X86_RESET16_TPL_DATA  = b'reset16tpl'
  60 PPC_MPC85XX_BR_DATA   = b'ppcmpc85xxbr'
  61 U_BOOT_NODTB_DATA     = b'nodtb with microcode pointer somewhere in here'
  62 U_BOOT_SPL_NODTB_DATA = b'splnodtb with microcode pointer somewhere in here'
  63 U_BOOT_TPL_NODTB_DATA = b'tplnodtb with microcode pointer somewhere in here'
  64 FSP_DATA              = b'fsp'
  65 CMC_DATA              = b'cmc'
  66 VBT_DATA              = b'vbt'
  67 MRC_DATA              = b'mrc'
  68 TEXT_DATA             = 'text'
  69 TEXT_DATA2            = 'text2'
  70 TEXT_DATA3            = 'text3'
  71 CROS_EC_RW_DATA       = b'ecrw'
  72 GBB_DATA              = b'gbbd'
  73 BMPBLK_DATA           = b'bmp'
  74 VBLOCK_DATA           = b'vblk'
  75 FILES_DATA            = (b"sorry I'm late\nOh, don't bother apologising, I'm " +
  76                          b"sorry you're alive\n")
  77 COMPRESS_DATA         = b'compress xxxxxxxxxxxxxxxxxxxxxx data'
  78 COMPRESS_DATA_BIG     = COMPRESS_DATA * 2
  79 REFCODE_DATA          = b'refcode'
  80 FSP_M_DATA            = b'fsp_m'
  81 FSP_S_DATA            = b'fsp_s'
  82 FSP_T_DATA            = b'fsp_t'
  83 ATF_BL31_DATA         = b'bl31'
  84 ATF_BL2U_DATA         = b'bl2u'
  85 OPENSBI_DATA          = b'opensbi'
  86 SCP_DATA              = b'scp'
  87 TEST_FDT1_DATA        = b'fdt1'
  88 TEST_FDT2_DATA        = b'test-fdt2'
  89 ENV_DATA              = b'var1=1\nvar2="2"'
  90
  91 # Subdirectory of the input dir to use to put test FDTs
  92 TEST_FDT_SUBDIR       = 'fdts'
  93
  94 # The expected size for the device tree in some tests
  95 EXTRACT_DTB_SIZE = 0x3c9
  96
  97 # Properties expected to be in the device tree when update_dtb is used
  98 BASE_DTB_PROPS = ['offset', 'size', 'image-pos']
  99
 100 # Extra properties expected to be in the device tree when allow-repack is used
 101 REPACK_DTB_PROPS = ['orig-offset', 'orig-size']
 102
 103
 104 class TestFunctional(unittest.TestCase):
 105     """Functional tests for binman
 106
 107     Most of these use a sample .dts file to build an image and then check
 108     that it looks correct. The sample files are in the test/ subdirectory
 109     and are numbered.
 110
 111     For each entry type a very small test file is created using fixed
 112     string contents. This makes it easy to test that things look right, and
 113     debug problems.
 114
 115     In some cases a 'real' file must be used - these are also supplied in
 116     the test/ diurectory.
 117     """
 118     @classmethod
 119     def setUpClass(cls):
 120         global entry
 121         from binman import entry
 122
 123         # Handle the case where argv[0] is 'python'
 124         cls._binman_dir = os.path.dirname(os.path.realpath(sys.argv[0]))
 125         cls._binman_pathname = os.path.join(cls._binman_dir, 'binman')
 126
 127         # Create a temporary directory for input files
 128         cls._indir = tempfile.mkdtemp(prefix='binmant.')
 129
 130         # Create some test files
 131         TestFunctional._MakeInputFile('u-boot.bin', U_BOOT_DATA)
 132         TestFunctional._MakeInputFile('u-boot.img', U_BOOT_IMG_DATA)
 133         TestFunctional._MakeInputFile('spl/u-boot-spl.bin', U_BOOT_SPL_DATA)
 134         TestFunctional._MakeInputFile('tpl/u-boot-tpl.bin', U_BOOT_TPL_DATA)
 135         TestFunctional._MakeInputFile('blobfile', BLOB_DATA)
 136         TestFunctional._MakeInputFile('me.bin', ME_DATA)
 137         TestFunctional._MakeInputFile('vga.bin', VGA_DATA)
 138         cls._ResetDtbs()
 139
 140         TestFunctional._MakeInputFile('u-boot-br.bin', PPC_MPC85XX_BR_DATA)
 141
 142         TestFunctional._MakeInputFile('u-boot-x86-start16.bin', X86_START16_DATA)
 143         TestFunctional._MakeInputFile('spl/u-boot-x86-start16-spl.bin',
 144                                       X86_START16_SPL_DATA)
 145         TestFunctional._MakeInputFile('tpl/u-boot-x86-start16-tpl.bin',
 146                                       X86_START16_TPL_DATA)
 147
 148         TestFunctional._MakeInputFile('u-boot-x86-reset16.bin',
 149                                       X86_RESET16_DATA)
 150         TestFunctional._MakeInputFile('spl/u-boot-x86-reset16-spl.bin',
 151                                       X86_RESET16_SPL_DATA)
 152         TestFunctional._MakeInputFile('tpl/u-boot-x86-reset16-tpl.bin',
 153                                       X86_RESET16_TPL_DATA)
 154
 155         TestFunctional._MakeInputFile('u-boot-nodtb.bin', U_BOOT_NODTB_DATA)
 156         TestFunctional._MakeInputFile('spl/u-boot-spl-nodtb.bin',
 157                                       U_BOOT_SPL_NODTB_DATA)
 158         TestFunctional._MakeInputFile('tpl/u-boot-tpl-nodtb.bin',
 159                                       U_BOOT_TPL_NODTB_DATA)
 160         TestFunctional._MakeInputFile('fsp.bin', FSP_DATA)
 161         TestFunctional._MakeInputFile('cmc.bin', CMC_DATA)
 162         TestFunctional._MakeInputFile('vbt.bin', VBT_DATA)
 163         TestFunctional._MakeInputFile('mrc.bin', MRC_DATA)
 164         TestFunctional._MakeInputFile('ecrw.bin', CROS_EC_RW_DATA)
 165         TestFunctional._MakeInputDir('devkeys')
 166         TestFunctional._MakeInputFile('bmpblk.bin', BMPBLK_DATA)
 167         TestFunctional._MakeInputFile('refcode.bin', REFCODE_DATA)
 168         TestFunctional._MakeInputFile('fsp_m.bin', FSP_M_DATA)
 169         TestFunctional._MakeInputFile('fsp_s.bin', FSP_S_DATA)
 170         TestFunctional._MakeInputFile('fsp_t.bin', FSP_T_DATA)
 171
 172         cls._elf_testdir = os.path.join(cls._indir, 'elftest')
 173         elf_test.BuildElfTestFiles(cls._elf_testdir)
 174
 175         # ELF file with a '_dt_ucode_base_size' symbol
 176         TestFunctional._MakeInputFile('u-boot',
 177             tools.read_file(cls.ElfTestFile('u_boot_ucode_ptr')))
 178
 179         # Intel flash descriptor file
 180         cls._SetupDescriptor()
 181
 182         shutil.copytree(cls.TestFile('files'),
 183                         os.path.join(cls._indir, 'files'))
 184
 185         TestFunctional._MakeInputFile('compress', COMPRESS_DATA)
 186         TestFunctional._MakeInputFile('compress_big', COMPRESS_DATA_BIG)
 187         TestFunctional._MakeInputFile('bl31.bin', ATF_BL31_DATA)
 188         TestFunctional._MakeInputFile('bl2u.bin', ATF_BL2U_DATA)
 189         TestFunctional._MakeInputFile('fw_dynamic.bin', OPENSBI_DATA)
 190         TestFunctional._MakeInputFile('scp.bin', SCP_DATA)
 191
 192         # Add a few .dtb files for testing
 193         TestFunctional._MakeInputFile('%s/test-fdt1.dtb' % TEST_FDT_SUBDIR,
 194                                       TEST_FDT1_DATA)
 195         TestFunctional._MakeInputFile('%s/test-fdt2.dtb' % TEST_FDT_SUBDIR,
 196                                       TEST_FDT2_DATA)
 197
 198         TestFunctional._MakeInputFile('env.txt', ENV_DATA)
 199
 200         cls.have_lz4 = comp_util.HAVE_LZ4
 201
 202     @classmethod
 203     def tearDownClass(cls):
 204         """Remove the temporary input directory and its contents"""
 205         if cls.preserve_indir:
 206             print('Preserving input dir: %s' % cls._indir)
 207         else:
 208             if cls._indir:
 209                 shutil.rmtree(cls._indir)
 210         cls._indir = None
 211
 212     @classmethod
 213     def setup_test_args(cls, preserve_indir=False, preserve_outdirs=False,
 214                         toolpath=None, verbosity=None):
 215         """Accept arguments controlling test execution
 216
 217         Args:
 218             preserve_indir: Preserve the shared input directory used by all
 219                 tests in this class.
 220             preserve_outdir: Preserve the output directories used by tests. Each
 221                 test has its own, so this is normally only useful when running a
 222                 single test.
 223             toolpath: ist of paths to use for tools
 224         """
 225         cls.preserve_indir = preserve_indir
 226         cls.preserve_outdirs = preserve_outdirs
 227         cls.toolpath = toolpath
 228         cls.verbosity = verbosity
 229
 230     def _CheckLz4(self):
 231         if not self.have_lz4:
 232             self.skipTest('lz4 --no-frame-crc not available')
 233
 234     def _CleanupOutputDir(self):
 235         """Remove the temporary output directory"""
 236         if self.preserve_outdirs:
 237             print('Preserving output dir: %s' % tools.outdir)
 238         else:
 239             tools._finalise_for_test()
 240
 241     def setUp(self):
 242         # Enable this to turn on debugging output
 243         # tout.Init(tout.DEBUG)
 244         command.test_result = None
 245
 246     def tearDown(self):
 247         """Remove the temporary output directory"""
 248         self._CleanupOutputDir()
 249
 250     def _SetupImageInTmpdir(self):
 251         """Set up the output image in a new temporary directory
 252
 253         This is used when an image has been generated in the output directory,
 254         but we want to run binman again. This will create a new output
 255         directory and fail to delete the original one.
 256
 257         This creates a new temporary directory, copies the image to it (with a
 258         new name) and removes the old output directory.
 259
 260         Returns:
 261             Tuple:
 262                 Temporary directory to use
 263                 New image filename
 264         """
 265         image_fname = tools.get_output_filename('image.bin')
 266         tmpdir = tempfile.mkdtemp(prefix='binman.')
 267         updated_fname = os.path.join(tmpdir, 'image-updated.bin')
 268         tools.write_file(updated_fname, tools.read_file(image_fname))
 269         self._CleanupOutputDir()
 270         return tmpdir, updated_fname
 271
 272     @classmethod
 273     def _ResetDtbs(cls):
 274         TestFunctional._MakeInputFile('u-boot.dtb', U_BOOT_DTB_DATA)
 275         TestFunctional._MakeInputFile('spl/u-boot-spl.dtb', U_BOOT_SPL_DTB_DATA)
 276         TestFunctional._MakeInputFile('tpl/u-boot-tpl.dtb', U_BOOT_TPL_DTB_DATA)
 277
 278     def _RunBinman(self, *args, **kwargs):
 279         """Run binman using the command line
 280
 281         Args:
 282             Arguments to pass, as a list of strings
 283             kwargs: Arguments to pass to Command.RunPipe()
 284         """
 285         result = command.run_pipe([[self._binman_pathname] + list(args)],
 286                 capture=True, capture_stderr=True, raise_on_error=False)
 287         if result.return_code and kwargs.get('raise_on_error', True):
 288             raise Exception("Error running '%s': %s" % (' '.join(args),
 289                             result.stdout + result.stderr))
 290         return result
 291
 292     def _DoBinman(self, *argv):
 293         """Run binman using directly (in the same process)
 294
 295         Args:
 296             Arguments to pass, as a list of strings
 297         Returns:
 298             Return value (0 for success)
 299         """
 300         argv = list(argv)
 301         args = cmdline.ParseArgs(argv)
 302         args.pager = 'binman-invalid-pager'
 303         args.build_dir = self._indir
 304
 305         # For testing, you can force an increase in verbosity here
 306         # args.verbosity = tout.DEBUG
 307         return control.Binman(args)
 308
 309     def _DoTestFile(self, fname, debug=False, map=False, update_dtb=False,
 310                     entry_args=None, images=None, use_real_dtb=False,
 311                     use_expanded=False, verbosity=None, allow_missing=False,
 312                     allow_fake_blobs=False, extra_indirs=None, threads=None,
 313                     test_section_timeout=False, update_fdt_in_elf=None,
 314                     force_missing_bintools=''):
 315         """Run binman with a given test file
 316
 317         Args:
 318             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 319             debug: True to enable debugging output
 320             map: True to output map files for the images
 321             update_dtb: Update the offset and size of each entry in the device
 322                 tree before packing it into the image
 323             entry_args: Dict of entry args to supply to binman
 324                 key: arg name
 325                 value: value of that arg
 326             images: List of image names to build
 327             use_real_dtb: True to use the test file as the contents of
 328                 the u-boot-dtb entry. Normally this is not needed and the
 329                 test contents (the U_BOOT_DTB_DATA string) can be used.
 330                 But in some test we need the real contents.
 331             use_expanded: True to use expanded entries where available, e.g.
 332                 'u-boot-expanded' instead of 'u-boot'
 333             verbosity: Verbosity level to use (0-3, None=don't set it)
 334             allow_missing: Set the '--allow-missing' flag so that missing
 335                 external binaries just produce a warning instead of an error
 336             allow_fake_blobs: Set the '--fake-ext-blobs' flag
 337             extra_indirs: Extra input directories to add using -I
 338             threads: Number of threads to use (None for default, 0 for
 339                 single-threaded)
 340             test_section_timeout: True to force the first time to timeout, as
 341                 used in testThreadTimeout()
 342             update_fdt_in_elf: Value to pass with --update-fdt-in-elf=xxx
 343             force_missing_tools (str): comma-separated list of bintools to
 344                 regard as missing
 345
 346         Returns:
 347             int return code, 0 on success
 348         """
 349         args = []
 350         if debug:
 351             args.append('-D')
 352         if verbosity is not None:
 353             args.append('-v%d' % verbosity)
 354         elif self.verbosity:
 355             args.append('-v%d' % self.verbosity)
 356         if self.toolpath:
 357             for path in self.toolpath:
 358                 args += ['--toolpath', path]
 359         if threads is not None:
 360             args.append('-T%d' % threads)
 361         if test_section_timeout:
 362             args.append('--test-section-timeout')
 363         args += ['build', '-p', '-I', self._indir, '-d', self.TestFile(fname)]
 364         if map:
 365             args.append('-m')
 366         if update_dtb:
 367             args.append('-u')
 368         if not use_real_dtb:
 369             args.append('--fake-dtb')
 370         if not use_expanded:
 371             args.append('--no-expanded')
 372         if entry_args:
 373             for arg, value in entry_args.items():
 374                 args.append('-a%s=%s' % (arg, value))
 375         if allow_missing:
 376             args.append('-M')
 377         if allow_fake_blobs:
 378             args.append('--fake-ext-blobs')
 379         if force_missing_bintools:
 380             args += ['--force-missing-bintools', force_missing_bintools]
 381         if update_fdt_in_elf:
 382             args += ['--update-fdt-in-elf', update_fdt_in_elf]
 383         if images:
 384             for image in images:
 385                 args += ['-i', image]
 386         if extra_indirs:
 387             for indir in extra_indirs:
 388                 args += ['-I', indir]
 389         return self._DoBinman(*args)
 390
 391     def _SetupDtb(self, fname, outfile='u-boot.dtb'):
 392         """Set up a new test device-tree file
 393
 394         The given file is compiled and set up as the device tree to be used
 395         for ths test.
 396
 397         Args:
 398             fname: Filename of .dts file to read
 399             outfile: Output filename for compiled device-tree binary
 400
 401         Returns:
 402             Contents of device-tree binary
 403         """
 404         tmpdir = tempfile.mkdtemp(prefix='binmant.')
 405         dtb = fdt_util.EnsureCompiled(self.TestFile(fname), tmpdir)
 406         with open(dtb, 'rb') as fd:
 407             data = fd.read()
 408             TestFunctional._MakeInputFile(outfile, data)
 409         shutil.rmtree(tmpdir)
 410         return data
 411
 412     def _GetDtbContentsForSplTpl(self, dtb_data, name):
 413         """Create a version of the main DTB for SPL or SPL
 414
 415         For testing we don't actually have different versions of the DTB. With
 416         U-Boot we normally run fdtgrep to remove unwanted nodes, but for tests
 417         we don't normally have any unwanted nodes.
 418
 419         We still want the DTBs for SPL and TPL to be different though, since
 420         otherwise it is confusing to know which one we are looking at. So add
 421         an 'spl' or 'tpl' property to the top-level node.
 422
 423         Args:
 424             dtb_data: dtb data to modify (this should be a value devicetree)
 425             name: Name of a new property to add
 426
 427         Returns:
 428             New dtb data with the property added
 429         """
 430         dtb = fdt.Fdt.FromData(dtb_data)
 431         dtb.Scan()
 432         dtb.GetNode('/binman').AddZeroProp(name)
 433         dtb.Sync(auto_resize=True)
 434         dtb.Pack()
 435         return dtb.GetContents()
 436
 437     def _DoReadFileDtb(self, fname, use_real_dtb=False, use_expanded=False,
 438                        map=False, update_dtb=False, entry_args=None,
 439                        reset_dtbs=True, extra_indirs=None, threads=None):
 440         """Run binman and return the resulting image
 441
 442         This runs binman with a given test file and then reads the resulting
 443         output file. It is a shortcut function since most tests need to do
 444         these steps.
 445
 446         Raises an assertion failure if binman returns a non-zero exit code.
 447
 448         Args:
 449             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 450             use_real_dtb: True to use the test file as the contents of
 451                 the u-boot-dtb entry. Normally this is not needed and the
 452                 test contents (the U_BOOT_DTB_DATA string) can be used.
 453                 But in some test we need the real contents.
 454             use_expanded: True to use expanded entries where available, e.g.
 455                 'u-boot-expanded' instead of 'u-boot'
 456             map: True to output map files for the images
 457             update_dtb: Update the offset and size of each entry in the device
 458                 tree before packing it into the image
 459             entry_args: Dict of entry args to supply to binman
 460                 key: arg name
 461                 value: value of that arg
 462             reset_dtbs: With use_real_dtb the test dtb is overwritten by this
 463                 function. If reset_dtbs is True, then the original test dtb
 464                 is written back before this function finishes
 465             extra_indirs: Extra input directories to add using -I
 466             threads: Number of threads to use (None for default, 0 for
 467                 single-threaded)
 468
 469         Returns:
 470             Tuple:
 471                 Resulting image contents
 472                 Device tree contents
 473                 Map data showing contents of image (or None if none)
 474                 Output device tree binary filename ('u-boot.dtb' path)
 475         """
 476         dtb_data = None
 477         # Use the compiled test file as the u-boot-dtb input
 478         if use_real_dtb:
 479             dtb_data = self._SetupDtb(fname)
 480
 481             # For testing purposes, make a copy of the DT for SPL and TPL. Add
 482             # a node indicating which it is, so aid verification.
 483             for name in ['spl', 'tpl']:
 484                 dtb_fname = '%s/u-boot-%s.dtb' % (name, name)
 485                 outfile = os.path.join(self._indir, dtb_fname)
 486                 TestFunctional._MakeInputFile(dtb_fname,
 487                         self._GetDtbContentsForSplTpl(dtb_data, name))
 488
 489         try:
 490             retcode = self._DoTestFile(fname, map=map, update_dtb=update_dtb,
 491                     entry_args=entry_args, use_real_dtb=use_real_dtb,
 492                     use_expanded=use_expanded, extra_indirs=extra_indirs,
 493                     threads=threads)
 494             self.assertEqual(0, retcode)
 495             out_dtb_fname = tools.get_output_filename('u-boot.dtb.out')
 496
 497             # Find the (only) image, read it and return its contents
 498             image = control.images['image']
 499             image_fname = tools.get_output_filename('image.bin')
 500             self.assertTrue(os.path.exists(image_fname))
 501             if map:
 502                 map_fname = tools.get_output_filename('image.map')
 503                 with open(map_fname) as fd:
 504                     map_data = fd.read()
 505             else:
 506                 map_data = None
 507             with open(image_fname, 'rb') as fd:
 508                 return fd.read(), dtb_data, map_data, out_dtb_fname
 509         finally:
 510             # Put the test file back
 511             if reset_dtbs and use_real_dtb:
 512                 self._ResetDtbs()
 513
 514     def _DoReadFileRealDtb(self, fname):
 515         """Run binman with a real .dtb file and return the resulting data
 516
 517         Args:
 518             fname: DT source filename to use (e.g. 082_fdt_update_all.dts)
 519
 520         Returns:
 521             Resulting image contents
 522         """
 523         return self._DoReadFileDtb(fname, use_real_dtb=True, update_dtb=True)[0]
 524
 525     def _DoReadFile(self, fname, use_real_dtb=False):
 526         """Helper function which discards the device-tree binary
 527
 528         Args:
 529             fname: Device-tree source filename to use (e.g. 005_simple.dts)
 530             use_real_dtb: True to use the test file as the contents of
 531                 the u-boot-dtb entry. Normally this is not needed and the
 532                 test contents (the U_BOOT_DTB_DATA string) can be used.
 533                 But in some test we need the real contents.
 534
 535         Returns:
 536             Resulting image contents
 537         """
 538         return self._DoReadFileDtb(fname, use_real_dtb)[0]
 539
 540     @classmethod
 541     def _MakeInputFile(cls, fname, contents):
 542         """Create a new test input file, creating directories as needed
 543
 544         Args:
 545             fname: Filename to create
 546             contents: File contents to write in to the file
 547         Returns:
 548             Full pathname of file created
 549         """
 550         pathname = os.path.join(cls._indir, fname)
 551         dirname = os.path.dirname(pathname)
 552         if dirname and not os.path.exists(dirname):
 553             os.makedirs(dirname)
 554         with open(pathname, 'wb') as fd:
 555             fd.write(contents)
 556         return pathname
 557
 558     @classmethod
 559     def _MakeInputDir(cls, dirname):
 560         """Create a new test input directory, creating directories as needed
 561
 562         Args:
 563             dirname: Directory name to create
 564
 565         Returns:
 566             Full pathname of directory created
 567         """
 568         pathname = os.path.join(cls._indir, dirname)
 569         if not os.path.exists(pathname):
 570             os.makedirs(pathname)
 571         return pathname
 572
 573     @classmethod
 574     def _SetupSplElf(cls, src_fname='bss_data'):
 575         """Set up an ELF file with a '_dt_ucode_base_size' symbol
 576
 577         Args:
 578             Filename of ELF file to use as SPL
 579         """
 580         TestFunctional._MakeInputFile('spl/u-boot-spl',
 581             tools.read_file(cls.ElfTestFile(src_fname)))
 582
 583     @classmethod
 584     def _SetupTplElf(cls, src_fname='bss_data'):
 585         """Set up an ELF file with a '_dt_ucode_base_size' symbol
 586
 587         Args:
 588             Filename of ELF file to use as TPL
 589         """
 590         TestFunctional._MakeInputFile('tpl/u-boot-tpl',
 591             tools.read_file(cls.ElfTestFile(src_fname)))
 592
 593     @classmethod
 594     def _SetupDescriptor(cls):
 595         with open(cls.TestFile('descriptor.bin'), 'rb') as fd:
 596             TestFunctional._MakeInputFile('descriptor.bin', fd.read())
 597
 598     @classmethod
 599     def TestFile(cls, fname):
 600         return os.path.join(cls._binman_dir, 'test', fname)
 601
 602     @classmethod
 603     def ElfTestFile(cls, fname):
 604         return os.path.join(cls._elf_testdir, fname)
 605
 606     def AssertInList(self, grep_list, target):
 607         """Assert that at least one of a list of things is in a target
 608
 609         Args:
 610             grep_list: List of strings to check
 611             target: Target string
 612         """
 613         for grep in grep_list:
 614             if grep in target:
 615                 return
 616         self.fail("Error: '%s' not found in '%s'" % (grep_list, target))
 617
 618     def CheckNoGaps(self, entries):
 619         """Check that all entries fit together without gaps
 620
 621         Args:
 622             entries: List of entries to check
 623         """
 624         offset = 0
 625         for entry in entries.values():
 626             self.assertEqual(offset, entry.offset)
 627             offset += entry.size
 628
 629     def GetFdtLen(self, dtb):
 630         """Get the totalsize field from a device-tree binary
 631
 632         Args:
 633             dtb: Device-tree binary contents
 634
 635         Returns:
 636             Total size of device-tree binary, from the header
 637         """
 638         return struct.unpack('>L', dtb[4:8])[0]
 639
 640     def _GetPropTree(self, dtb, prop_names, prefix='/binman/'):
 641         def AddNode(node, path):
 642             if node.name != '/':
 643                 path += '/' + node.name
 644             for prop in node.props.values():
 645                 if prop.name in prop_names:
 646                     prop_path = path + ':' + prop.name
 647                     tree[prop_path[len(prefix):]] = fdt_util.fdt32_to_cpu(
 648                         prop.value)
 649             for subnode in node.subnodes:
 650                 AddNode(subnode, path)
 651
 652         tree = {}
 653         AddNode(dtb.GetRoot(), '')
 654         return tree
 655
 656     def testRun(self):
 657         """Test a basic run with valid args"""
 658         result = self._RunBinman('-h')
 659
 660     def testFullHelp(self):
 661         """Test that the full help is displayed with -H"""
 662         result = self._RunBinman('-H')
 663         help_file = os.path.join(self._binman_dir, 'README.rst')
 664         # Remove possible extraneous strings
 665         extra = '::::::::::::::\n' + help_file + '\n::::::::::::::\n'
 666         gothelp = result.stdout.replace(extra, '')
 667         self.assertEqual(len(gothelp), os.path.getsize(help_file))
 668         self.assertEqual(0, len(result.stderr))
 669         self.assertEqual(0, result.return_code)
 670
 671     def testFullHelpInternal(self):
 672         """Test that the full help is displayed with -H"""
 673         try:
 674             command.test_result = command.CommandResult()
 675             result = self._DoBinman('-H')
 676             help_file = os.path.join(self._binman_dir, 'README.rst')
 677         finally:
 678             command.test_result = None
 679
 680     def testHelp(self):
 681         """Test that the basic help is displayed with -h"""
 682         result = self._RunBinman('-h')
 683         self.assertTrue(len(result.stdout) > 200)
 684         self.assertEqual(0, len(result.stderr))
 685         self.assertEqual(0, result.return_code)
 686
 687     def testBoard(self):
 688         """Test that we can run it with a specific board"""
 689         self._SetupDtb('005_simple.dts', 'sandbox/u-boot.dtb')
 690         TestFunctional._MakeInputFile('sandbox/u-boot.bin', U_BOOT_DATA)
 691         result = self._DoBinman('build', '-n', '-b', 'sandbox')
 692         self.assertEqual(0, result)
 693
 694     def testNeedBoard(self):
 695         """Test that we get an error when no board ius supplied"""
 696         with self.assertRaises(ValueError) as e:
 697             result = self._DoBinman('build')
 698         self.assertIn("Must provide a board to process (use -b <board>)",
 699                 str(e.exception))
 700
 701     def testMissingDt(self):
 702         """Test that an invalid device-tree file generates an error"""
 703         with self.assertRaises(Exception) as e:
 704             self._RunBinman('build', '-d', 'missing_file')
 705         # We get one error from libfdt, and a different one from fdtget.
 706         self.AssertInList(["Couldn't open blob from 'missing_file'",
 707                            'No such file or directory'], str(e.exception))
 708
 709     def testBrokenDt(self):
 710         """Test that an invalid device-tree source file generates an error
 711
 712         Since this is a source file it should be compiled and the error
 713         will come from the device-tree compiler (dtc).
 714         """
 715         with self.assertRaises(Exception) as e:
 716             self._RunBinman('build', '-d', self.TestFile('001_invalid.dts'))
 717         self.assertIn("FATAL ERROR: Unable to parse input tree",
 718                 str(e.exception))
 719
 720     def testMissingNode(self):
 721         """Test that a device tree without a 'binman' node generates an error"""
 722         with self.assertRaises(Exception) as e:
 723             self._DoBinman('build', '-d', self.TestFile('002_missing_node.dts'))
 724         self.assertIn("does not have a 'binman' node", str(e.exception))
 725
 726     def testEmpty(self):
 727         """Test that an empty binman node works OK (i.e. does nothing)"""
 728         result = self._RunBinman('build', '-d', self.TestFile('003_empty.dts'))
 729         self.assertEqual(0, len(result.stderr))
 730         self.assertEqual(0, result.return_code)
 731
 732     def testInvalidEntry(self):
 733         """Test that an invalid entry is flagged"""
 734         with self.assertRaises(Exception) as e:
 735             result = self._RunBinman('build', '-d',
 736                                      self.TestFile('004_invalid_entry.dts'))
 737         self.assertIn("Unknown entry type 'not-a-valid-type' in node "
 738                 "'/binman/not-a-valid-type'", str(e.exception))
 739
 740     def testSimple(self):
 741         """Test a simple binman with a single file"""
 742         data = self._DoReadFile('005_simple.dts')
 743         self.assertEqual(U_BOOT_DATA, data)
 744
 745     def testSimpleDebug(self):
 746         """Test a simple binman run with debugging enabled"""
 747         self._DoTestFile('005_simple.dts', debug=True)
 748
 749     def testDual(self):
 750         """Test that we can handle creating two images
 751
 752         This also tests image padding.
 753         """
 754         retcode = self._DoTestFile('006_dual_image.dts')
 755         self.assertEqual(0, retcode)
 756
 757         image = control.images['image1']
 758         self.assertEqual(len(U_BOOT_DATA), image.size)
 759         fname = tools.get_output_filename('image1.bin')
 760         self.assertTrue(os.path.exists(fname))
 761         with open(fname, 'rb') as fd:
 762             data = fd.read()
 763             self.assertEqual(U_BOOT_DATA, data)
 764
 765         image = control.images['image2']
 766         self.assertEqual(3 + len(U_BOOT_DATA) + 5, image.size)
 767         fname = tools.get_output_filename('image2.bin')
 768         self.assertTrue(os.path.exists(fname))
 769         with open(fname, 'rb') as fd:
 770             data = fd.read()
 771             self.assertEqual(U_BOOT_DATA, data[3:7])
 772             self.assertEqual(tools.get_bytes(0, 3), data[:3])
 773             self.assertEqual(tools.get_bytes(0, 5), data[7:])
 774
 775     def testBadAlign(self):
 776         """Test that an invalid alignment value is detected"""
 777         with self.assertRaises(ValueError) as e:
 778             self._DoTestFile('007_bad_align.dts')
 779         self.assertIn("Node '/binman/u-boot': Alignment 23 must be a power "
 780                       "of two", str(e.exception))
 781
 782     def testPackSimple(self):
 783         """Test that packing works as expected"""
 784         retcode = self._DoTestFile('008_pack.dts')
 785         self.assertEqual(0, retcode)
 786         self.assertIn('image', control.images)
 787         image = control.images['image']
 788         entries = image.GetEntries()
 789         self.assertEqual(5, len(entries))
 790
 791         # First u-boot
 792         self.assertIn('u-boot', entries)
 793         entry = entries['u-boot']
 794         self.assertEqual(0, entry.offset)
 795         self.assertEqual(len(U_BOOT_DATA), entry.size)
 796
 797         # Second u-boot, aligned to 16-byte boundary
 798         self.assertIn('u-boot-align', entries)
 799         entry = entries['u-boot-align']
 800         self.assertEqual(16, entry.offset)
 801         self.assertEqual(len(U_BOOT_DATA), entry.size)
 802
 803         # Third u-boot, size 23 bytes
 804         self.assertIn('u-boot-size', entries)
 805         entry = entries['u-boot-size']
 806         self.assertEqual(20, entry.offset)
 807         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
 808         self.assertEqual(23, entry.size)
 809
 810         # Fourth u-boot, placed immediate after the above
 811         self.assertIn('u-boot-next', entries)
 812         entry = entries['u-boot-next']
 813         self.assertEqual(43, entry.offset)
 814         self.assertEqual(len(U_BOOT_DATA), entry.size)
 815
 816         # Fifth u-boot, placed at a fixed offset
 817         self.assertIn('u-boot-fixed', entries)
 818         entry = entries['u-boot-fixed']
 819         self.assertEqual(61, entry.offset)
 820         self.assertEqual(len(U_BOOT_DATA), entry.size)
 821
 822         self.assertEqual(65, image.size)
 823
 824     def testPackExtra(self):
 825         """Test that extra packing feature works as expected"""
 826         data, _, _, out_dtb_fname = self._DoReadFileDtb('009_pack_extra.dts',
 827                                                         update_dtb=True)
 828
 829         self.assertIn('image', control.images)
 830         image = control.images['image']
 831         entries = image.GetEntries()
 832         self.assertEqual(5, len(entries))
 833
 834         # First u-boot with padding before and after
 835         self.assertIn('u-boot', entries)
 836         entry = entries['u-boot']
 837         self.assertEqual(0, entry.offset)
 838         self.assertEqual(3, entry.pad_before)
 839         self.assertEqual(3 + 5 + len(U_BOOT_DATA), entry.size)
 840         self.assertEqual(U_BOOT_DATA, entry.data)
 841         self.assertEqual(tools.get_bytes(0, 3) + U_BOOT_DATA +
 842                          tools.get_bytes(0, 5), data[:entry.size])
 843         pos = entry.size
 844
 845         # Second u-boot has an aligned size, but it has no effect
 846         self.assertIn('u-boot-align-size-nop', entries)
 847         entry = entries['u-boot-align-size-nop']
 848         self.assertEqual(pos, entry.offset)
 849         self.assertEqual(len(U_BOOT_DATA), entry.size)
 850         self.assertEqual(U_BOOT_DATA, entry.data)
 851         self.assertEqual(U_BOOT_DATA, data[pos:pos + entry.size])
 852         pos += entry.size
 853
 854         # Third u-boot has an aligned size too
 855         self.assertIn('u-boot-align-size', entries)
 856         entry = entries['u-boot-align-size']
 857         self.assertEqual(pos, entry.offset)
 858         self.assertEqual(32, entry.size)
 859         self.assertEqual(U_BOOT_DATA, entry.data)
 860         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 32 - len(U_BOOT_DATA)),
 861                          data[pos:pos + entry.size])
 862         pos += entry.size
 863
 864         # Fourth u-boot has an aligned end
 865         self.assertIn('u-boot-align-end', entries)
 866         entry = entries['u-boot-align-end']
 867         self.assertEqual(48, entry.offset)
 868         self.assertEqual(16, entry.size)
 869         self.assertEqual(U_BOOT_DATA, entry.data[:len(U_BOOT_DATA)])
 870         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 16 - len(U_BOOT_DATA)),
 871                          data[pos:pos + entry.size])
 872         pos += entry.size
 873
 874         # Fifth u-boot immediately afterwards
 875         self.assertIn('u-boot-align-both', entries)
 876         entry = entries['u-boot-align-both']
 877         self.assertEqual(64, entry.offset)
 878         self.assertEqual(64, entry.size)
 879         self.assertEqual(U_BOOT_DATA, entry.data[:len(U_BOOT_DATA)])
 880         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 64 - len(U_BOOT_DATA)),
 881                          data[pos:pos + entry.size])
 882
 883         self.CheckNoGaps(entries)
 884         self.assertEqual(128, image.size)
 885
 886         dtb = fdt.Fdt(out_dtb_fname)
 887         dtb.Scan()
 888         props = self._GetPropTree(dtb, ['size', 'offset', 'image-pos'])
 889         expected = {
 890             'image-pos': 0,
 891             'offset': 0,
 892             'size': 128,
 893
 894             'u-boot:image-pos': 0,
 895             'u-boot:offset': 0,
 896             'u-boot:size': 3 + 5 + len(U_BOOT_DATA),
 897
 898             'u-boot-align-size-nop:image-pos': 12,
 899             'u-boot-align-size-nop:offset': 12,
 900             'u-boot-align-size-nop:size': 4,
 901
 902             'u-boot-align-size:image-pos': 16,
 903             'u-boot-align-size:offset': 16,
 904             'u-boot-align-size:size': 32,
 905
 906             'u-boot-align-end:image-pos': 48,
 907             'u-boot-align-end:offset': 48,
 908             'u-boot-align-end:size': 16,
 909
 910             'u-boot-align-both:image-pos': 64,
 911             'u-boot-align-both:offset': 64,
 912             'u-boot-align-both:size': 64,
 913             }
 914         self.assertEqual(expected, props)
 915
 916     def testPackAlignPowerOf2(self):
 917         """Test that invalid entry alignment is detected"""
 918         with self.assertRaises(ValueError) as e:
 919             self._DoTestFile('010_pack_align_power2.dts')
 920         self.assertIn("Node '/binman/u-boot': Alignment 5 must be a power "
 921                       "of two", str(e.exception))
 922
 923     def testPackAlignSizePowerOf2(self):
 924         """Test that invalid entry size alignment is detected"""
 925         with self.assertRaises(ValueError) as e:
 926             self._DoTestFile('011_pack_align_size_power2.dts')
 927         self.assertIn("Node '/binman/u-boot': Alignment size 55 must be a "
 928                       "power of two", str(e.exception))
 929
 930     def testPackInvalidAlign(self):
 931         """Test detection of an offset that does not match its alignment"""
 932         with self.assertRaises(ValueError) as e:
 933             self._DoTestFile('012_pack_inv_align.dts')
 934         self.assertIn("Node '/binman/u-boot': Offset 0x5 (5) does not match "
 935                       "align 0x4 (4)", str(e.exception))
 936
 937     def testPackInvalidSizeAlign(self):
 938         """Test that invalid entry size alignment is detected"""
 939         with self.assertRaises(ValueError) as e:
 940             self._DoTestFile('013_pack_inv_size_align.dts')
 941         self.assertIn("Node '/binman/u-boot': Size 0x5 (5) does not match "
 942                       "align-size 0x4 (4)", str(e.exception))
 943
 944     def testPackOverlap(self):
 945         """Test that overlapping regions are detected"""
 946         with self.assertRaises(ValueError) as e:
 947             self._DoTestFile('014_pack_overlap.dts')
 948         self.assertIn("Node '/binman/u-boot-align': Offset 0x3 (3) overlaps "
 949                       "with previous entry '/binman/u-boot' ending at 0x4 (4)",
 950                       str(e.exception))
 951
 952     def testPackEntryOverflow(self):
 953         """Test that entries that overflow their size are detected"""
 954         with self.assertRaises(ValueError) as e:
 955             self._DoTestFile('015_pack_overflow.dts')
 956         self.assertIn("Node '/binman/u-boot': Entry contents size is 0x4 (4) "
 957                       "but entry size is 0x3 (3)", str(e.exception))
 958
 959     def testPackImageOverflow(self):
 960         """Test that entries which overflow the image size are detected"""
 961         with self.assertRaises(ValueError) as e:
 962             self._DoTestFile('016_pack_image_overflow.dts')
 963         self.assertIn("Section '/binman': contents size 0x4 (4) exceeds section "
 964                       "size 0x3 (3)", str(e.exception))
 965
 966     def testPackImageSize(self):
 967         """Test that the image size can be set"""
 968         retcode = self._DoTestFile('017_pack_image_size.dts')
 969         self.assertEqual(0, retcode)
 970         self.assertIn('image', control.images)
 971         image = control.images['image']
 972         self.assertEqual(7, image.size)
 973
 974     def testPackImageSizeAlign(self):
 975         """Test that image size alignemnt works as expected"""
 976         retcode = self._DoTestFile('018_pack_image_align.dts')
 977         self.assertEqual(0, retcode)
 978         self.assertIn('image', control.images)
 979         image = control.images['image']
 980         self.assertEqual(16, image.size)
 981
 982     def testPackInvalidImageAlign(self):
 983         """Test that invalid image alignment is detected"""
 984         with self.assertRaises(ValueError) as e:
 985             self._DoTestFile('019_pack_inv_image_align.dts')
 986         self.assertIn("Section '/binman': Size 0x7 (7) does not match "
 987                       "align-size 0x8 (8)", str(e.exception))
 988
 989     def testPackAlignPowerOf2(self):
 990         """Test that invalid image alignment is detected"""
 991         with self.assertRaises(ValueError) as e:
 992             self._DoTestFile('020_pack_inv_image_align_power2.dts')
 993         self.assertIn("Image '/binman': Alignment size 131 must be a power of "
 994                       "two", str(e.exception))
 995
 996     def testImagePadByte(self):
 997         """Test that the image pad byte can be specified"""
 998         self._SetupSplElf()
 999         data = self._DoReadFile('021_image_pad.dts')
1000         self.assertEqual(U_BOOT_SPL_DATA + tools.get_bytes(0xff, 1) +
1001                          U_BOOT_DATA, data)
1002
1003     def testImageName(self):
1004         """Test that image files can be named"""
1005         retcode = self._DoTestFile('022_image_name.dts')
1006         self.assertEqual(0, retcode)
1007         image = control.images['image1']
1008         fname = tools.get_output_filename('test-name')
1009         self.assertTrue(os.path.exists(fname))
1010
1011         image = control.images['image2']
1012         fname = tools.get_output_filename('test-name.xx')
1013         self.assertTrue(os.path.exists(fname))
1014
1015     def testBlobFilename(self):
1016         """Test that generic blobs can be provided by filename"""
1017         data = self._DoReadFile('023_blob.dts')
1018         self.assertEqual(BLOB_DATA, data)
1019
1020     def testPackSorted(self):
1021         """Test that entries can be sorted"""
1022         self._SetupSplElf()
1023         data = self._DoReadFile('024_sorted.dts')
1024         self.assertEqual(tools.get_bytes(0, 1) + U_BOOT_SPL_DATA +
1025                          tools.get_bytes(0, 2) + U_BOOT_DATA, data)
1026
1027     def testPackZeroOffset(self):
1028         """Test that an entry at offset 0 is not given a new offset"""
1029         with self.assertRaises(ValueError) as e:
1030             self._DoTestFile('025_pack_zero_size.dts')
1031         self.assertIn("Node '/binman/u-boot-spl': Offset 0x0 (0) overlaps "
1032                       "with previous entry '/binman/u-boot' ending at 0x4 (4)",
1033                       str(e.exception))
1034
1035     def testPackUbootDtb(self):
1036         """Test that a device tree can be added to U-Boot"""
1037         data = self._DoReadFile('026_pack_u_boot_dtb.dts')
1038         self.assertEqual(U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA, data)
1039
1040     def testPackX86RomNoSize(self):
1041         """Test that the end-at-4gb property requires a size property"""
1042         with self.assertRaises(ValueError) as e:
1043             self._DoTestFile('027_pack_4gb_no_size.dts')
1044         self.assertIn("Image '/binman': Section size must be provided when "
1045                       "using end-at-4gb", str(e.exception))
1046
1047     def test4gbAndSkipAtStartTogether(self):
1048         """Test that the end-at-4gb and skip-at-size property can't be used
1049         together"""
1050         with self.assertRaises(ValueError) as e:
1051             self._DoTestFile('098_4gb_and_skip_at_start_together.dts')
1052         self.assertIn("Image '/binman': Provide either 'end-at-4gb' or "
1053                       "'skip-at-start'", str(e.exception))
1054
1055     def testPackX86RomOutside(self):
1056         """Test that the end-at-4gb property checks for offset boundaries"""
1057         with self.assertRaises(ValueError) as e:
1058             self._DoTestFile('028_pack_4gb_outside.dts')
1059         self.assertIn("Node '/binman/u-boot': Offset 0x0 (0) size 0x4 (4) "
1060                       "is outside the section '/binman' starting at "
1061                       '0xffffffe0 (4294967264) of size 0x20 (32)',
1062                       str(e.exception))
1063
1064     def testPackX86Rom(self):
1065         """Test that a basic x86 ROM can be created"""
1066         self._SetupSplElf()
1067         data = self._DoReadFile('029_x86_rom.dts')
1068         self.assertEqual(U_BOOT_DATA + tools.get_bytes(0, 3) + U_BOOT_SPL_DATA +
1069                          tools.get_bytes(0, 2), data)
1070
1071     def testPackX86RomMeNoDesc(self):
1072         """Test that an invalid Intel descriptor entry is detected"""
1073         try:
1074             TestFunctional._MakeInputFile('descriptor-empty.bin', b'')
1075             with self.assertRaises(ValueError) as e:
1076                 self._DoTestFile('163_x86_rom_me_empty.dts')
1077             self.assertIn("Node '/binman/intel-descriptor': Cannot find Intel Flash Descriptor (FD) signature",
1078                           str(e.exception))
1079         finally:
1080             self._SetupDescriptor()
1081
1082     def testPackX86RomBadDesc(self):
1083         """Test that the Intel requires a descriptor entry"""
1084         with self.assertRaises(ValueError) as e:
1085             self._DoTestFile('030_x86_rom_me_no_desc.dts')
1086         self.assertIn("Node '/binman/intel-me': No offset set with "
1087                       "offset-unset: should another entry provide this correct "
1088                       "offset?", str(e.exception))
1089
1090     def testPackX86RomMe(self):
1091         """Test that an x86 ROM with an ME region can be created"""
1092         data = self._DoReadFile('031_x86_rom_me.dts')
1093         expected_desc = tools.read_file(self.TestFile('descriptor.bin'))
1094         if data[:0x1000] != expected_desc:
1095             self.fail('Expected descriptor binary at start of image')
1096         self.assertEqual(ME_DATA, data[0x1000:0x1000 + len(ME_DATA)])
1097
1098     def testPackVga(self):
1099         """Test that an image with a VGA binary can be created"""
1100         data = self._DoReadFile('032_intel_vga.dts')
1101         self.assertEqual(VGA_DATA, data[:len(VGA_DATA)])
1102
1103     def testPackStart16(self):
1104         """Test that an image with an x86 start16 region can be created"""
1105         data = self._DoReadFile('033_x86_start16.dts')
1106         self.assertEqual(X86_START16_DATA, data[:len(X86_START16_DATA)])
1107
1108     def testPackPowerpcMpc85xxBootpgResetvec(self):
1109         """Test that an image with powerpc-mpc85xx-bootpg-resetvec can be
1110         created"""
1111         data = self._DoReadFile('150_powerpc_mpc85xx_bootpg_resetvec.dts')
1112         self.assertEqual(PPC_MPC85XX_BR_DATA, data[:len(PPC_MPC85XX_BR_DATA)])
1113
1114     def _RunMicrocodeTest(self, dts_fname, nodtb_data, ucode_second=False):
1115         """Handle running a test for insertion of microcode
1116
1117         Args:
1118             dts_fname: Name of test .dts file
1119             nodtb_data: Data that we expect in the first section
1120             ucode_second: True if the microsecond entry is second instead of
1121                 third
1122
1123         Returns:
1124             Tuple:
1125                 Contents of first region (U-Boot or SPL)
1126                 Offset and size components of microcode pointer, as inserted
1127                     in the above (two 4-byte words)
1128         """
1129         data = self._DoReadFile(dts_fname, True)
1130
1131         # Now check the device tree has no microcode
1132         if ucode_second:
1133             ucode_content = data[len(nodtb_data):]
1134             ucode_pos = len(nodtb_data)
1135             dtb_with_ucode = ucode_content[16:]
1136             fdt_len = self.GetFdtLen(dtb_with_ucode)
1137         else:
1138             dtb_with_ucode = data[len(nodtb_data):]
1139             fdt_len = self.GetFdtLen(dtb_with_ucode)
1140             ucode_content = dtb_with_ucode[fdt_len:]
1141             ucode_pos = len(nodtb_data) + fdt_len
1142         fname = tools.get_output_filename('test.dtb')
1143         with open(fname, 'wb') as fd:
1144             fd.write(dtb_with_ucode)
1145         dtb = fdt.FdtScan(fname)
1146         ucode = dtb.GetNode('/microcode')
1147         self.assertTrue(ucode)
1148         for node in ucode.subnodes:
1149             self.assertFalse(node.props.get('data'))
1150
1151         # Check that the microcode appears immediately after the Fdt
1152         # This matches the concatenation of the data properties in
1153         # the /microcode/update@xxx nodes in 34_x86_ucode.dts.
1154         ucode_data = struct.pack('>4L', 0x12345678, 0x12345679, 0xabcd0000,
1155                                  0x78235609)
1156         self.assertEqual(ucode_data, ucode_content[:len(ucode_data)])
1157
1158         # Check that the microcode pointer was inserted. It should match the
1159         # expected offset and size
1160         pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos,
1161                                    len(ucode_data))
1162         u_boot = data[:len(nodtb_data)]
1163         return u_boot, pos_and_size
1164
1165     def testPackUbootMicrocode(self):
1166         """Test that x86 microcode can be handled correctly
1167
1168         We expect to see the following in the image, in order:
1169             u-boot-nodtb.bin with a microcode pointer inserted at the correct
1170                 place
1171             u-boot.dtb with the microcode removed
1172             the microcode
1173         """
1174         first, pos_and_size = self._RunMicrocodeTest('034_x86_ucode.dts',
1175                                                      U_BOOT_NODTB_DATA)
1176         self.assertEqual(b'nodtb with microcode' + pos_and_size +
1177                          b' somewhere in here', first)
1178
1179     def _RunPackUbootSingleMicrocode(self):
1180         """Test that x86 microcode can be handled correctly
1181
1182         We expect to see the following in the image, in order:
1183             u-boot-nodtb.bin with a microcode pointer inserted at the correct
1184                 place
1185             u-boot.dtb with the microcode
1186             an empty microcode region
1187         """
1188         # We need the libfdt library to run this test since only that allows
1189         # finding the offset of a property. This is required by
1190         # Entry_u_boot_dtb_with_ucode.ObtainContents().
1191         data = self._DoReadFile('035_x86_single_ucode.dts', True)
1192
1193         second = data[len(U_BOOT_NODTB_DATA):]
1194
1195         fdt_len = self.GetFdtLen(second)
1196         third = second[fdt_len:]
1197         second = second[:fdt_len]
1198
1199         ucode_data = struct.pack('>2L', 0x12345678, 0x12345679)
1200         self.assertIn(ucode_data, second)
1201         ucode_pos = second.find(ucode_data) + len(U_BOOT_NODTB_DATA)
1202
1203         # Check that the microcode pointer was inserted. It should match the
1204         # expected offset and size
1205         pos_and_size = struct.pack('<2L', 0xfffffe00 + ucode_pos,
1206                                    len(ucode_data))
1207         first = data[:len(U_BOOT_NODTB_DATA)]
1208         self.assertEqual(b'nodtb with microcode' + pos_and_size +
1209                          b' somewhere in here', first)
1210
1211     def testPackUbootSingleMicrocode(self):
1212         """Test that x86 microcode can be handled correctly with fdt_normal.
1213         """
1214         self._RunPackUbootSingleMicrocode()
1215
1216     def testUBootImg(self):
1217         """Test that u-boot.img can be put in a file"""
1218         data = self._DoReadFile('036_u_boot_img.dts')
1219         self.assertEqual(U_BOOT_IMG_DATA, data)
1220
1221     def testNoMicrocode(self):
1222         """Test that a missing microcode region is detected"""
1223         with self.assertRaises(ValueError) as e:
1224             self._DoReadFile('037_x86_no_ucode.dts', True)
1225         self.assertIn("Node '/binman/u-boot-dtb-with-ucode': No /microcode "
1226                       "node found in ", str(e.exception))
1227
1228     def testMicrocodeWithoutNode(self):
1229         """Test that a missing u-boot-dtb-with-ucode node is detected"""
1230         with self.assertRaises(ValueError) as e:
1231             self._DoReadFile('038_x86_ucode_missing_node.dts', True)
1232         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot find "
1233                 "microcode region u-boot-dtb-with-ucode", str(e.exception))
1234
1235     def testMicrocodeWithoutNode2(self):
1236         """Test that a missing u-boot-ucode node is detected"""
1237         with self.assertRaises(ValueError) as e:
1238             self._DoReadFile('039_x86_ucode_missing_node2.dts', True)
1239         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot find "
1240             "microcode region u-boot-ucode", str(e.exception))
1241
1242     def testMicrocodeWithoutPtrInElf(self):
1243         """Test that a U-Boot binary without the microcode symbol is detected"""
1244         # ELF file without a '_dt_ucode_base_size' symbol
1245         try:
1246             TestFunctional._MakeInputFile('u-boot',
1247                 tools.read_file(self.ElfTestFile('u_boot_no_ucode_ptr')))
1248
1249             with self.assertRaises(ValueError) as e:
1250                 self._RunPackUbootSingleMicrocode()
1251             self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Cannot locate "
1252                     "_dt_ucode_base_size symbol in u-boot", str(e.exception))
1253
1254         finally:
1255             # Put the original file back
1256             TestFunctional._MakeInputFile('u-boot',
1257                 tools.read_file(self.ElfTestFile('u_boot_ucode_ptr')))
1258
1259     def testMicrocodeNotInImage(self):
1260         """Test that microcode must be placed within the image"""
1261         with self.assertRaises(ValueError) as e:
1262             self._DoReadFile('040_x86_ucode_not_in_image.dts', True)
1263         self.assertIn("Node '/binman/u-boot-with-ucode-ptr': Microcode "
1264                 "pointer _dt_ucode_base_size at fffffe14 is outside the "
1265                 "section ranging from 00000000 to 0000002e", str(e.exception))
1266
1267     def testWithoutMicrocode(self):
1268         """Test that we can cope with an image without microcode (e.g. qemu)"""
1269         TestFunctional._MakeInputFile('u-boot',
1270             tools.read_file(self.ElfTestFile('u_boot_no_ucode_ptr')))
1271         data, dtb, _, _ = self._DoReadFileDtb('044_x86_optional_ucode.dts', True)
1272
1273         # Now check the device tree has no microcode
1274         self.assertEqual(U_BOOT_NODTB_DATA, data[:len(U_BOOT_NODTB_DATA)])
1275         second = data[len(U_BOOT_NODTB_DATA):]
1276
1277         fdt_len = self.GetFdtLen(second)
1278         self.assertEqual(dtb, second[:fdt_len])
1279
1280         used_len = len(U_BOOT_NODTB_DATA) + fdt_len
1281         third = data[used_len:]
1282         self.assertEqual(tools.get_bytes(0, 0x200 - used_len), third)
1283
1284     def testUnknownPosSize(self):
1285         """Test that microcode must be placed within the image"""
1286         with self.assertRaises(ValueError) as e:
1287             self._DoReadFile('041_unknown_pos_size.dts', True)
1288         self.assertIn("Section '/binman': Unable to set offset/size for unknown "
1289                 "entry 'invalid-entry'", str(e.exception))
1290
1291     def testPackFsp(self):
1292         """Test that an image with a FSP binary can be created"""
1293         data = self._DoReadFile('042_intel_fsp.dts')
1294         self.assertEqual(FSP_DATA, data[:len(FSP_DATA)])
1295
1296     def testPackCmc(self):
1297         """Test that an image with a CMC binary can be created"""
1298         data = self._DoReadFile('043_intel_cmc.dts')
1299         self.assertEqual(CMC_DATA, data[:len(CMC_DATA)])
1300
1301     def testPackVbt(self):
1302         """Test that an image with a VBT binary can be created"""
1303         data = self._DoReadFile('046_intel_vbt.dts')
1304         self.assertEqual(VBT_DATA, data[:len(VBT_DATA)])
1305
1306     def testSplBssPad(self):
1307         """Test that we can pad SPL's BSS with zeros"""
1308         # ELF file with a '__bss_size' symbol
1309         self._SetupSplElf()
1310         data = self._DoReadFile('047_spl_bss_pad.dts')
1311         self.assertEqual(U_BOOT_SPL_DATA + tools.get_bytes(0, 10) + U_BOOT_DATA,
1312                          data)
1313
1314     def testSplBssPadMissing(self):
1315         """Test that a missing symbol is detected"""
1316         self._SetupSplElf('u_boot_ucode_ptr')
1317         with self.assertRaises(ValueError) as e:
1318             self._DoReadFile('047_spl_bss_pad.dts')
1319         self.assertIn('Expected __bss_size symbol in spl/u-boot-spl',
1320                       str(e.exception))
1321
1322     def testPackStart16Spl(self):
1323         """Test that an image with an x86 start16 SPL region can be created"""
1324         data = self._DoReadFile('048_x86_start16_spl.dts')
1325         self.assertEqual(X86_START16_SPL_DATA, data[:len(X86_START16_SPL_DATA)])
1326
1327     def _PackUbootSplMicrocode(self, dts, ucode_second=False):
1328         """Helper function for microcode tests
1329
1330         We expect to see the following in the image, in order:
1331             u-boot-spl-nodtb.bin with a microcode pointer inserted at the
1332                 correct place
1333             u-boot.dtb with the microcode removed
1334             the microcode
1335
1336         Args:
1337             dts: Device tree file to use for test
1338             ucode_second: True if the microsecond entry is second instead of
1339                 third
1340         """
1341         self._SetupSplElf('u_boot_ucode_ptr')
1342         first, pos_and_size = self._RunMicrocodeTest(dts, U_BOOT_SPL_NODTB_DATA,
1343                                                      ucode_second=ucode_second)
1344         self.assertEqual(b'splnodtb with microc' + pos_and_size +
1345                          b'ter somewhere in here', first)
1346
1347     def testPackUbootSplMicrocode(self):
1348         """Test that x86 microcode can be handled correctly in SPL"""
1349         self._PackUbootSplMicrocode('049_x86_ucode_spl.dts')
1350
1351     def testPackUbootSplMicrocodeReorder(self):
1352         """Test that order doesn't matter for microcode entries
1353
1354         This is the same as testPackUbootSplMicrocode but when we process the
1355         u-boot-ucode entry we have not yet seen the u-boot-dtb-with-ucode
1356         entry, so we reply on binman to try later.
1357         """
1358         self._PackUbootSplMicrocode('058_x86_ucode_spl_needs_retry.dts',
1359                                     ucode_second=True)
1360
1361     def testPackMrc(self):
1362         """Test that an image with an MRC binary can be created"""
1363         data = self._DoReadFile('050_intel_mrc.dts')
1364         self.assertEqual(MRC_DATA, data[:len(MRC_DATA)])
1365
1366     def testSplDtb(self):
1367         """Test that an image with spl/u-boot-spl.dtb can be created"""
1368         data = self._DoReadFile('051_u_boot_spl_dtb.dts')
1369         self.assertEqual(U_BOOT_SPL_DTB_DATA, data[:len(U_BOOT_SPL_DTB_DATA)])
1370
1371     def testSplNoDtb(self):
1372         """Test that an image with spl/u-boot-spl-nodtb.bin can be created"""
1373         self._SetupSplElf()
1374         data = self._DoReadFile('052_u_boot_spl_nodtb.dts')
1375         self.assertEqual(U_BOOT_SPL_NODTB_DATA, data[:len(U_BOOT_SPL_NODTB_DATA)])
1376
1377     def checkSymbols(self, dts, base_data, u_boot_offset, entry_args=None,
1378                      use_expanded=False):
1379         """Check the image contains the expected symbol values
1380
1381         Args:
1382             dts: Device tree file to use for test
1383             base_data: Data before and after 'u-boot' section
1384             u_boot_offset: Offset of 'u-boot' section in image
1385             entry_args: Dict of entry args to supply to binman
1386                 key: arg name
1387                 value: value of that arg
1388             use_expanded: True to use expanded entries where available, e.g.
1389                 'u-boot-expanded' instead of 'u-boot'
1390         """
1391         elf_fname = self.ElfTestFile('u_boot_binman_syms')
1392         syms = elf.GetSymbols(elf_fname, ['binman', 'image'])
1393         addr = elf.GetSymbolAddress(elf_fname, '__image_copy_start')
1394         self.assertEqual(syms['_binman_u_boot_spl_any_prop_offset'].address,
1395                          addr)
1396
1397         self._SetupSplElf('u_boot_binman_syms')
1398         data = self._DoReadFileDtb(dts, entry_args=entry_args,
1399                                    use_expanded=use_expanded)[0]
1400         # The image should contain the symbols from u_boot_binman_syms.c
1401         # Note that image_pos is adjusted by the base address of the image,
1402         # which is 0x10 in our test image
1403         sym_values = struct.pack('<LQLL', 0x00,
1404                                  u_boot_offset + len(U_BOOT_DATA),
1405                                  0x10 + u_boot_offset, 0x04)
1406         expected = (sym_values + base_data[20:] +
1407                     tools.get_bytes(0xff, 1) + U_BOOT_DATA + sym_values +
1408                     base_data[20:])
1409         self.assertEqual(expected, data)
1410
1411     def testSymbols(self):
1412         """Test binman can assign symbols embedded in U-Boot"""
1413         self.checkSymbols('053_symbols.dts', U_BOOT_SPL_DATA, 0x18)
1414
1415     def testSymbolsNoDtb(self):
1416         """Test binman can assign symbols embedded in U-Boot SPL"""
1417         self.checkSymbols('196_symbols_nodtb.dts',
1418                           U_BOOT_SPL_NODTB_DATA + U_BOOT_SPL_DTB_DATA,
1419                           0x38)
1420
1421     def testPackUnitAddress(self):
1422         """Test that we support multiple binaries with the same name"""
1423         data = self._DoReadFile('054_unit_address.dts')
1424         self.assertEqual(U_BOOT_DATA + U_BOOT_DATA, data)
1425
1426     def testSections(self):
1427         """Basic test of sections"""
1428         data = self._DoReadFile('055_sections.dts')
1429         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
1430                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12) +
1431                     U_BOOT_DATA + tools.get_bytes(ord('&'), 4))
1432         self.assertEqual(expected, data)
1433
1434     def testMap(self):
1435         """Tests outputting a map of the images"""
1436         _, _, map_data, _ = self._DoReadFileDtb('055_sections.dts', map=True)
1437         self.assertEqual('''ImagePos    Offset      Size  Name
1438 00000000  00000000  00000028  main-section
1439 00000000   00000000  00000010  section@0
1440 00000000    00000000  00000004  u-boot
1441 00000010   00000010  00000010  section@1
1442 00000010    00000000  00000004  u-boot
1443 00000020   00000020  00000004  section@2
1444 00000020    00000000  00000004  u-boot
1445 ''', map_data)
1446
1447     def testNamePrefix(self):
1448         """Tests that name prefixes are used"""
1449         _, _, map_data, _ = self._DoReadFileDtb('056_name_prefix.dts', map=True)
1450         self.assertEqual('''ImagePos    Offset      Size  Name
1451 00000000  00000000  00000028  main-section
1452 00000000   00000000  00000010  section@0
1453 00000000    00000000  00000004  ro-u-boot
1454 00000010   00000010  00000010  section@1
1455 00000010    00000000  00000004  rw-u-boot
1456 ''', map_data)
1457
1458     def testUnknownContents(self):
1459         """Test that obtaining the contents works as expected"""
1460         with self.assertRaises(ValueError) as e:
1461             self._DoReadFile('057_unknown_contents.dts', True)
1462         self.assertIn("Image '/binman': Internal error: Could not complete "
1463                 "processing of contents: remaining ["
1464                 "<binman.etype._testing.Entry__testing ", str(e.exception))
1465
1466     def testBadChangeSize(self):
1467         """Test that trying to change the size of an entry fails"""
1468         try:
1469             state.SetAllowEntryExpansion(False)
1470             with self.assertRaises(ValueError) as e:
1471                 self._DoReadFile('059_change_size.dts', True)
1472             self.assertIn("Node '/binman/_testing': Cannot update entry size from 2 to 3",
1473                           str(e.exception))
1474         finally:
1475             state.SetAllowEntryExpansion(True)
1476
1477     def testUpdateFdt(self):
1478         """Test that we can update the device tree with offset/size info"""
1479         _, _, _, out_dtb_fname = self._DoReadFileDtb('060_fdt_update.dts',
1480                                                      update_dtb=True)
1481         dtb = fdt.Fdt(out_dtb_fname)
1482         dtb.Scan()
1483         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
1484         self.assertEqual({
1485             'image-pos': 0,
1486             'offset': 0,
1487             '_testing:offset': 32,
1488             '_testing:size': 2,
1489             '_testing:image-pos': 32,
1490             'section@0/u-boot:offset': 0,
1491             'section@0/u-boot:size': len(U_BOOT_DATA),
1492             'section@0/u-boot:image-pos': 0,
1493             'section@0:offset': 0,
1494             'section@0:size': 16,
1495             'section@0:image-pos': 0,
1496
1497             'section@1/u-boot:offset': 0,
1498             'section@1/u-boot:size': len(U_BOOT_DATA),
1499             'section@1/u-boot:image-pos': 16,
1500             'section@1:offset': 16,
1501             'section@1:size': 16,
1502             'section@1:image-pos': 16,
1503             'size': 40
1504         }, props)
1505
1506     def testUpdateFdtBad(self):
1507         """Test that we detect when ProcessFdt never completes"""
1508         with self.assertRaises(ValueError) as e:
1509             self._DoReadFileDtb('061_fdt_update_bad.dts', update_dtb=True)
1510         self.assertIn('Could not complete processing of Fdt: remaining '
1511                       '[<binman.etype._testing.Entry__testing',
1512                         str(e.exception))
1513
1514     def testEntryArgs(self):
1515         """Test passing arguments to entries from the command line"""
1516         entry_args = {
1517             'test-str-arg': 'test1',
1518             'test-int-arg': '456',
1519         }
1520         self._DoReadFileDtb('062_entry_args.dts', entry_args=entry_args)
1521         self.assertIn('image', control.images)
1522         entry = control.images['image'].GetEntries()['_testing']
1523         self.assertEqual('test0', entry.test_str_fdt)
1524         self.assertEqual('test1', entry.test_str_arg)
1525         self.assertEqual(123, entry.test_int_fdt)
1526         self.assertEqual(456, entry.test_int_arg)
1527
1528     def testEntryArgsMissing(self):
1529         """Test missing arguments and properties"""
1530         entry_args = {
1531             'test-int-arg': '456',
1532         }
1533         self._DoReadFileDtb('063_entry_args_missing.dts', entry_args=entry_args)
1534         entry = control.images['image'].GetEntries()['_testing']
1535         self.assertEqual('test0', entry.test_str_fdt)
1536         self.assertEqual(None, entry.test_str_arg)
1537         self.assertEqual(None, entry.test_int_fdt)
1538         self.assertEqual(456, entry.test_int_arg)
1539
1540     def testEntryArgsRequired(self):
1541         """Test missing arguments and properties"""
1542         entry_args = {
1543             'test-int-arg': '456',
1544         }
1545         with self.assertRaises(ValueError) as e:
1546             self._DoReadFileDtb('064_entry_args_required.dts')
1547         self.assertIn("Node '/binman/_testing': "
1548             'Missing required properties/entry args: test-str-arg, '
1549             'test-int-fdt, test-int-arg',
1550             str(e.exception))
1551
1552     def testEntryArgsInvalidFormat(self):
1553         """Test that an invalid entry-argument format is detected"""
1554         args = ['build', '-d', self.TestFile('064_entry_args_required.dts'),
1555                 '-ano-value']
1556         with self.assertRaises(ValueError) as e:
1557             self._DoBinman(*args)
1558         self.assertIn("Invalid entry arguemnt 'no-value'", str(e.exception))
1559
1560     def testEntryArgsInvalidInteger(self):
1561         """Test that an invalid entry-argument integer is detected"""
1562         entry_args = {
1563             'test-int-arg': 'abc',
1564         }
1565         with self.assertRaises(ValueError) as e:
1566             self._DoReadFileDtb('062_entry_args.dts', entry_args=entry_args)
1567         self.assertIn("Node '/binman/_testing': Cannot convert entry arg "
1568                       "'test-int-arg' (value 'abc') to integer",
1569             str(e.exception))
1570
1571     def testEntryArgsInvalidDatatype(self):
1572         """Test that an invalid entry-argument datatype is detected
1573
1574         This test could be written in entry_test.py except that it needs
1575         access to control.entry_args, which seems more than that module should
1576         be able to see.
1577         """
1578         entry_args = {
1579             'test-bad-datatype-arg': '12',
1580         }
1581         with self.assertRaises(ValueError) as e:
1582             self._DoReadFileDtb('065_entry_args_unknown_datatype.dts',
1583                                 entry_args=entry_args)
1584         self.assertIn('GetArg() internal error: Unknown data type ',
1585                       str(e.exception))
1586
1587     def testText(self):
1588         """Test for a text entry type"""
1589         entry_args = {
1590             'test-id': TEXT_DATA,
1591             'test-id2': TEXT_DATA2,
1592             'test-id3': TEXT_DATA3,
1593         }
1594         data, _, _, _ = self._DoReadFileDtb('066_text.dts',
1595                                             entry_args=entry_args)
1596         expected = (tools.to_bytes(TEXT_DATA) +
1597                     tools.get_bytes(0, 8 - len(TEXT_DATA)) +
1598                     tools.to_bytes(TEXT_DATA2) + tools.to_bytes(TEXT_DATA3) +
1599                     b'some text' + b'more text')
1600         self.assertEqual(expected, data)
1601
1602     def testEntryDocs(self):
1603         """Test for creation of entry documentation"""
1604         with test_util.capture_sys_output() as (stdout, stderr):
1605             control.WriteEntryDocs(control.GetEntryModules())
1606         self.assertTrue(len(stdout.getvalue()) > 0)
1607
1608     def testEntryDocsMissing(self):
1609         """Test handling of missing entry documentation"""
1610         with self.assertRaises(ValueError) as e:
1611             with test_util.capture_sys_output() as (stdout, stderr):
1612                 control.WriteEntryDocs(control.GetEntryModules(), 'u_boot')
1613         self.assertIn('Documentation is missing for modules: u_boot',
1614                       str(e.exception))
1615
1616     def testFmap(self):
1617         """Basic test of generation of a flashrom fmap"""
1618         data = self._DoReadFile('067_fmap.dts')
1619         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
1620         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
1621                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12))
1622         self.assertEqual(expected, data[:32])
1623         self.assertEqual(b'__FMAP__', fhdr.signature)
1624         self.assertEqual(1, fhdr.ver_major)
1625         self.assertEqual(0, fhdr.ver_minor)
1626         self.assertEqual(0, fhdr.base)
1627         expect_size = fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 5
1628         self.assertEqual(16 + 16 + expect_size, fhdr.image_size)
1629         self.assertEqual(b'FMAP', fhdr.name)
1630         self.assertEqual(5, fhdr.nareas)
1631         fiter = iter(fentries)
1632
1633         fentry = next(fiter)
1634         self.assertEqual(b'SECTION0', fentry.name)
1635         self.assertEqual(0, fentry.offset)
1636         self.assertEqual(16, fentry.size)
1637         self.assertEqual(0, fentry.flags)
1638
1639         fentry = next(fiter)
1640         self.assertEqual(b'RO_U_BOOT', fentry.name)
1641         self.assertEqual(0, fentry.offset)
1642         self.assertEqual(4, fentry.size)
1643         self.assertEqual(0, fentry.flags)
1644
1645         fentry = next(fiter)
1646         self.assertEqual(b'SECTION1', fentry.name)
1647         self.assertEqual(16, fentry.offset)
1648         self.assertEqual(16, fentry.size)
1649         self.assertEqual(0, fentry.flags)
1650
1651         fentry = next(fiter)
1652         self.assertEqual(b'RW_U_BOOT', fentry.name)
1653         self.assertEqual(16, fentry.offset)
1654         self.assertEqual(4, fentry.size)
1655         self.assertEqual(0, fentry.flags)
1656
1657         fentry = next(fiter)
1658         self.assertEqual(b'FMAP', fentry.name)
1659         self.assertEqual(32, fentry.offset)
1660         self.assertEqual(expect_size, fentry.size)
1661         self.assertEqual(0, fentry.flags)
1662
1663     def testBlobNamedByArg(self):
1664         """Test we can add a blob with the filename coming from an entry arg"""
1665         entry_args = {
1666             'cros-ec-rw-path': 'ecrw.bin',
1667         }
1668         self._DoReadFileDtb('068_blob_named_by_arg.dts', entry_args=entry_args)
1669
1670     def testFill(self):
1671         """Test for an fill entry type"""
1672         data = self._DoReadFile('069_fill.dts')
1673         expected = tools.get_bytes(0xff, 8) + tools.get_bytes(0, 8)
1674         self.assertEqual(expected, data)
1675
1676     def testFillNoSize(self):
1677         """Test for an fill entry type with no size"""
1678         with self.assertRaises(ValueError) as e:
1679             self._DoReadFile('070_fill_no_size.dts')
1680         self.assertIn("'fill' entry must have a size property",
1681                       str(e.exception))
1682
1683     def _HandleGbbCommand(self, pipe_list):
1684         """Fake calls to the futility utility"""
1685         if pipe_list[0][0] == 'futility':
1686             fname = pipe_list[0][-1]
1687             # Append our GBB data to the file, which will happen every time the
1688             # futility command is called.
1689             with open(fname, 'ab') as fd:
1690                 fd.write(GBB_DATA)
1691             return command.CommandResult()
1692
1693     def testGbb(self):
1694         """Test for the Chromium OS Google Binary Block"""
1695         command.test_result = self._HandleGbbCommand
1696         entry_args = {
1697             'keydir': 'devkeys',
1698             'bmpblk': 'bmpblk.bin',
1699         }
1700         data, _, _, _ = self._DoReadFileDtb('071_gbb.dts', entry_args=entry_args)
1701
1702         # Since futility
1703         expected = (GBB_DATA + GBB_DATA + tools.get_bytes(0, 8) +
1704                     tools.get_bytes(0, 0x2180 - 16))
1705         self.assertEqual(expected, data)
1706
1707     def testGbbTooSmall(self):
1708         """Test for the Chromium OS Google Binary Block being large enough"""
1709         with self.assertRaises(ValueError) as e:
1710             self._DoReadFileDtb('072_gbb_too_small.dts')
1711         self.assertIn("Node '/binman/gbb': GBB is too small",
1712                       str(e.exception))
1713
1714     def testGbbNoSize(self):
1715         """Test for the Chromium OS Google Binary Block having a size"""
1716         with self.assertRaises(ValueError) as e:
1717             self._DoReadFileDtb('073_gbb_no_size.dts')
1718         self.assertIn("Node '/binman/gbb': GBB must have a fixed size",
1719                       str(e.exception))
1720
1721     def testGbbMissing(self):
1722         """Test that binman still produces an image if futility is missing"""
1723         entry_args = {
1724             'keydir': 'devkeys',
1725         }
1726         with test_util.capture_sys_output() as (_, stderr):
1727             self._DoTestFile('071_gbb.dts', force_missing_bintools='futility',
1728                              entry_args=entry_args)
1729         err = stderr.getvalue()
1730         self.assertRegex(err,
1731                          "Image 'main-section'.*missing bintools.*: futility")
1732
1733     def _HandleVblockCommand(self, pipe_list):
1734         """Fake calls to the futility utility
1735
1736         The expected pipe is:
1737
1738            [('futility', 'vbutil_firmware', '--vblock',
1739              'vblock.vblock', '--keyblock', 'devkeys/firmware.keyblock',
1740              '--signprivate', 'devkeys/firmware_data_key.vbprivk',
1741              '--version', '1', '--fv', 'input.vblock', '--kernelkey',
1742              'devkeys/kernel_subkey.vbpubk', '--flags', '1')]
1743
1744         This writes to the output file (here, 'vblock.vblock'). If
1745         self._hash_data is False, it writes VBLOCK_DATA, else it writes a hash
1746         of the input data (here, 'input.vblock').
1747         """
1748         if pipe_list[0][0] == 'futility':
1749             fname = pipe_list[0][3]
1750             with open(fname, 'wb') as fd:
1751                 if self._hash_data:
1752                     infile = pipe_list[0][11]
1753                     m = hashlib.sha256()
1754                     data = tools.read_file(infile)
1755                     m.update(data)
1756                     fd.write(m.digest())
1757                 else:
1758                     fd.write(VBLOCK_DATA)
1759
1760             return command.CommandResult()
1761
1762     def testVblock(self):
1763         """Test for the Chromium OS Verified Boot Block"""
1764         self._hash_data = False
1765         command.test_result = self._HandleVblockCommand
1766         entry_args = {
1767             'keydir': 'devkeys',
1768         }
1769         data, _, _, _ = self._DoReadFileDtb('074_vblock.dts',
1770                                             entry_args=entry_args)
1771         expected = U_BOOT_DATA + VBLOCK_DATA + U_BOOT_DTB_DATA
1772         self.assertEqual(expected, data)
1773
1774     def testVblockNoContent(self):
1775         """Test we detect a vblock which has no content to sign"""
1776         with self.assertRaises(ValueError) as e:
1777             self._DoReadFile('075_vblock_no_content.dts')
1778         self.assertIn("Node '/binman/vblock': Collection must have a 'content' "
1779                       'property', str(e.exception))
1780
1781     def testVblockBadPhandle(self):
1782         """Test that we detect a vblock with an invalid phandle in contents"""
1783         with self.assertRaises(ValueError) as e:
1784             self._DoReadFile('076_vblock_bad_phandle.dts')
1785         self.assertIn("Node '/binman/vblock': Cannot find node for phandle "
1786                       '1000', str(e.exception))
1787
1788     def testVblockBadEntry(self):
1789         """Test that we detect an entry that points to a non-entry"""
1790         with self.assertRaises(ValueError) as e:
1791             self._DoReadFile('077_vblock_bad_entry.dts')
1792         self.assertIn("Node '/binman/vblock': Cannot find entry for node "
1793                       "'other'", str(e.exception))
1794
1795     def testVblockContent(self):
1796         """Test that the vblock signs the right data"""
1797         self._hash_data = True
1798         command.test_result = self._HandleVblockCommand
1799         entry_args = {
1800             'keydir': 'devkeys',
1801         }
1802         data = self._DoReadFileDtb(
1803             '189_vblock_content.dts', use_real_dtb=True, update_dtb=True,
1804             entry_args=entry_args)[0]
1805         hashlen = 32  # SHA256 hash is 32 bytes
1806         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
1807         hashval = data[-hashlen:]
1808         dtb = data[len(U_BOOT_DATA):-hashlen]
1809
1810         expected_data = U_BOOT_DATA + dtb
1811
1812         # The hashval should be a hash of the dtb
1813         m = hashlib.sha256()
1814         m.update(expected_data)
1815         expected_hashval = m.digest()
1816         self.assertEqual(expected_hashval, hashval)
1817
1818     def testVblockMissing(self):
1819         """Test that binman still produces an image if futility is missing"""
1820         entry_args = {
1821             'keydir': 'devkeys',
1822         }
1823         with test_util.capture_sys_output() as (_, stderr):
1824             self._DoTestFile('074_vblock.dts',
1825                              force_missing_bintools='futility',
1826                              entry_args=entry_args)
1827         err = stderr.getvalue()
1828         self.assertRegex(err,
1829                          "Image 'main-section'.*missing bintools.*: futility")
1830
1831     def testTpl(self):
1832         """Test that an image with TPL and its device tree can be created"""
1833         # ELF file with a '__bss_size' symbol
1834         self._SetupTplElf()
1835         data = self._DoReadFile('078_u_boot_tpl.dts')
1836         self.assertEqual(U_BOOT_TPL_DATA + U_BOOT_TPL_DTB_DATA, data)
1837
1838     def testUsesPos(self):
1839         """Test that the 'pos' property cannot be used anymore"""
1840         with self.assertRaises(ValueError) as e:
1841            data = self._DoReadFile('079_uses_pos.dts')
1842         self.assertIn("Node '/binman/u-boot': Please use 'offset' instead of "
1843                       "'pos'", str(e.exception))
1844
1845     def testFillZero(self):
1846         """Test for an fill entry type with a size of 0"""
1847         data = self._DoReadFile('080_fill_empty.dts')
1848         self.assertEqual(tools.get_bytes(0, 16), data)
1849
1850     def testTextMissing(self):
1851         """Test for a text entry type where there is no text"""
1852         with self.assertRaises(ValueError) as e:
1853             self._DoReadFileDtb('066_text.dts',)
1854         self.assertIn("Node '/binman/text': No value provided for text label "
1855                       "'test-id'", str(e.exception))
1856
1857     def testPackStart16Tpl(self):
1858         """Test that an image with an x86 start16 TPL region can be created"""
1859         data = self._DoReadFile('081_x86_start16_tpl.dts')
1860         self.assertEqual(X86_START16_TPL_DATA, data[:len(X86_START16_TPL_DATA)])
1861
1862     def testSelectImage(self):
1863         """Test that we can select which images to build"""
1864         expected = 'Skipping images: image1'
1865
1866         # We should only get the expected message in verbose mode
1867         for verbosity in (0, 2):
1868             with test_util.capture_sys_output() as (stdout, stderr):
1869                 retcode = self._DoTestFile('006_dual_image.dts',
1870                                            verbosity=verbosity,
1871                                            images=['image2'])
1872             self.assertEqual(0, retcode)
1873             if verbosity:
1874                 self.assertIn(expected, stdout.getvalue())
1875             else:
1876                 self.assertNotIn(expected, stdout.getvalue())
1877
1878             self.assertFalse(os.path.exists(tools.get_output_filename('image1.bin')))
1879             self.assertTrue(os.path.exists(tools.get_output_filename('image2.bin')))
1880             self._CleanupOutputDir()
1881
1882     def testUpdateFdtAll(self):
1883         """Test that all device trees are updated with offset/size info"""
1884         data = self._DoReadFileRealDtb('082_fdt_update_all.dts')
1885
1886         base_expected = {
1887             'section:image-pos': 0,
1888             'u-boot-tpl-dtb:size': 513,
1889             'u-boot-spl-dtb:size': 513,
1890             'u-boot-spl-dtb:offset': 493,
1891             'image-pos': 0,
1892             'section/u-boot-dtb:image-pos': 0,
1893             'u-boot-spl-dtb:image-pos': 493,
1894             'section/u-boot-dtb:size': 493,
1895             'u-boot-tpl-dtb:image-pos': 1006,
1896             'section/u-boot-dtb:offset': 0,
1897             'section:size': 493,
1898             'offset': 0,
1899             'section:offset': 0,
1900             'u-boot-tpl-dtb:offset': 1006,
1901             'size': 1519
1902         }
1903
1904         # We expect three device-tree files in the output, one after the other.
1905         # Read them in sequence. We look for an 'spl' property in the SPL tree,
1906         # and 'tpl' in the TPL tree, to make sure they are distinct from the
1907         # main U-Boot tree. All three should have the same postions and offset.
1908         start = 0
1909         for item in ['', 'spl', 'tpl']:
1910             dtb = fdt.Fdt.FromData(data[start:])
1911             dtb.Scan()
1912             props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS +
1913                                       ['spl', 'tpl'])
1914             expected = dict(base_expected)
1915             if item:
1916                 expected[item] = 0
1917             self.assertEqual(expected, props)
1918             start += dtb._fdt_obj.totalsize()
1919
1920     def testUpdateFdtOutput(self):
1921         """Test that output DTB files are updated"""
1922         try:
1923             data, dtb_data, _, _ = self._DoReadFileDtb('082_fdt_update_all.dts',
1924                     use_real_dtb=True, update_dtb=True, reset_dtbs=False)
1925
1926             # Unfortunately, compiling a source file always results in a file
1927             # called source.dtb (see fdt_util.EnsureCompiled()). The test
1928             # source file (e.g. test/075_fdt_update_all.dts) thus does not enter
1929             # binman as a file called u-boot.dtb. To fix this, copy the file
1930             # over to the expected place.
1931             start = 0
1932             for fname in ['u-boot.dtb.out', 'spl/u-boot-spl.dtb.out',
1933                           'tpl/u-boot-tpl.dtb.out']:
1934                 dtb = fdt.Fdt.FromData(data[start:])
1935                 size = dtb._fdt_obj.totalsize()
1936                 pathname = tools.get_output_filename(os.path.split(fname)[1])
1937                 outdata = tools.read_file(pathname)
1938                 name = os.path.split(fname)[0]
1939
1940                 if name:
1941                     orig_indata = self._GetDtbContentsForSplTpl(dtb_data, name)
1942                 else:
1943                     orig_indata = dtb_data
1944                 self.assertNotEqual(outdata, orig_indata,
1945                         "Expected output file '%s' be updated" % pathname)
1946                 self.assertEqual(outdata, data[start:start + size],
1947                         "Expected output file '%s' to match output image" %
1948                         pathname)
1949                 start += size
1950         finally:
1951             self._ResetDtbs()
1952
1953     def _decompress(self, data):
1954         return comp_util.decompress(data, 'lz4')
1955
1956     def testCompress(self):
1957         """Test compression of blobs"""
1958         self._CheckLz4()
1959         data, _, _, out_dtb_fname = self._DoReadFileDtb('083_compress.dts',
1960                                             use_real_dtb=True, update_dtb=True)
1961         dtb = fdt.Fdt(out_dtb_fname)
1962         dtb.Scan()
1963         props = self._GetPropTree(dtb, ['size', 'uncomp-size'])
1964         orig = self._decompress(data)
1965         self.assertEquals(COMPRESS_DATA, orig)
1966
1967         # Do a sanity check on various fields
1968         image = control.images['image']
1969         entries = image.GetEntries()
1970         self.assertEqual(1, len(entries))
1971
1972         entry = entries['blob']
1973         self.assertEqual(COMPRESS_DATA, entry.uncomp_data)
1974         self.assertEqual(len(COMPRESS_DATA), entry.uncomp_size)
1975         orig = self._decompress(entry.data)
1976         self.assertEqual(orig, entry.uncomp_data)
1977
1978         self.assertEqual(image.data, entry.data)
1979
1980         expected = {
1981             'blob:uncomp-size': len(COMPRESS_DATA),
1982             'blob:size': len(data),
1983             'size': len(data),
1984             }
1985         self.assertEqual(expected, props)
1986
1987     def testFiles(self):
1988         """Test bringing in multiple files"""
1989         data = self._DoReadFile('084_files.dts')
1990         self.assertEqual(FILES_DATA, data)
1991
1992     def testFilesCompress(self):
1993         """Test bringing in multiple files and compressing them"""
1994         self._CheckLz4()
1995         data = self._DoReadFile('085_files_compress.dts')
1996
1997         image = control.images['image']
1998         entries = image.GetEntries()
1999         files = entries['files']
2000         entries = files._entries
2001
2002         orig = b''
2003         for i in range(1, 3):
2004             key = '%d.dat' % i
2005             start = entries[key].image_pos
2006             len = entries[key].size
2007             chunk = data[start:start + len]
2008             orig += self._decompress(chunk)
2009
2010         self.assertEqual(FILES_DATA, orig)
2011
2012     def testFilesMissing(self):
2013         """Test missing files"""
2014         with self.assertRaises(ValueError) as e:
2015             data = self._DoReadFile('086_files_none.dts')
2016         self.assertIn("Node '/binman/files': Pattern \'files/*.none\' matched "
2017                       'no files', str(e.exception))
2018
2019     def testFilesNoPattern(self):
2020         """Test missing files"""
2021         with self.assertRaises(ValueError) as e:
2022             data = self._DoReadFile('087_files_no_pattern.dts')
2023         self.assertIn("Node '/binman/files': Missing 'pattern' property",
2024                       str(e.exception))
2025
2026     def testExpandSize(self):
2027         """Test an expanding entry"""
2028         data, _, map_data, _ = self._DoReadFileDtb('088_expand_size.dts',
2029                                                    map=True)
2030         expect = (tools.get_bytes(ord('a'), 8) + U_BOOT_DATA +
2031                   MRC_DATA + tools.get_bytes(ord('b'), 1) + U_BOOT_DATA +
2032                   tools.get_bytes(ord('c'), 8) + U_BOOT_DATA +
2033                   tools.get_bytes(ord('d'), 8))
2034         self.assertEqual(expect, data)
2035         self.assertEqual('''ImagePos    Offset      Size  Name
2036 00000000  00000000  00000028  main-section
2037 00000000   00000000  00000008  fill
2038 00000008   00000008  00000004  u-boot
2039 0000000c   0000000c  00000004  section
2040 0000000c    00000000  00000003  intel-mrc
2041 00000010   00000010  00000004  u-boot2
2042 00000014   00000014  0000000c  section2
2043 00000014    00000000  00000008  fill
2044 0000001c    00000008  00000004  u-boot
2045 00000020   00000020  00000008  fill2
2046 ''', map_data)
2047
2048     def testExpandSizeBad(self):
2049         """Test an expanding entry which fails to provide contents"""
2050         with test_util.capture_sys_output() as (stdout, stderr):
2051             with self.assertRaises(ValueError) as e:
2052                 self._DoReadFileDtb('089_expand_size_bad.dts', map=True)
2053         self.assertIn("Node '/binman/_testing': Cannot obtain contents when "
2054                       'expanding entry', str(e.exception))
2055
2056     def testHash(self):
2057         """Test hashing of the contents of an entry"""
2058         _, _, _, out_dtb_fname = self._DoReadFileDtb('090_hash.dts',
2059                 use_real_dtb=True, update_dtb=True)
2060         dtb = fdt.Fdt(out_dtb_fname)
2061         dtb.Scan()
2062         hash_node = dtb.GetNode('/binman/u-boot/hash').props['value']
2063         m = hashlib.sha256()
2064         m.update(U_BOOT_DATA)
2065         self.assertEqual(m.digest(), b''.join(hash_node.value))
2066
2067     def testHashNoAlgo(self):
2068         with self.assertRaises(ValueError) as e:
2069             self._DoReadFileDtb('091_hash_no_algo.dts', update_dtb=True)
2070         self.assertIn("Node \'/binman/u-boot\': Missing \'algo\' property for "
2071                       'hash node', str(e.exception))
2072
2073     def testHashBadAlgo(self):
2074         with self.assertRaises(ValueError) as e:
2075             self._DoReadFileDtb('092_hash_bad_algo.dts', update_dtb=True)
2076         self.assertIn("Node '/binman/u-boot': Unknown hash algorithm",
2077                       str(e.exception))
2078
2079     def testHashSection(self):
2080         """Test hashing of the contents of an entry"""
2081         _, _, _, out_dtb_fname = self._DoReadFileDtb('099_hash_section.dts',
2082                 use_real_dtb=True, update_dtb=True)
2083         dtb = fdt.Fdt(out_dtb_fname)
2084         dtb.Scan()
2085         hash_node = dtb.GetNode('/binman/section/hash').props['value']
2086         m = hashlib.sha256()
2087         m.update(U_BOOT_DATA)
2088         m.update(tools.get_bytes(ord('a'), 16))
2089         self.assertEqual(m.digest(), b''.join(hash_node.value))
2090
2091     def testPackUBootTplMicrocode(self):
2092         """Test that x86 microcode can be handled correctly in TPL
2093
2094         We expect to see the following in the image, in order:
2095             u-boot-tpl-nodtb.bin with a microcode pointer inserted at the correct
2096                 place
2097             u-boot-tpl.dtb with the microcode removed
2098             the microcode
2099         """
2100         self._SetupTplElf('u_boot_ucode_ptr')
2101         first, pos_and_size = self._RunMicrocodeTest('093_x86_tpl_ucode.dts',
2102                                                      U_BOOT_TPL_NODTB_DATA)
2103         self.assertEqual(b'tplnodtb with microc' + pos_and_size +
2104                          b'ter somewhere in here', first)
2105
2106     def testFmapX86(self):
2107         """Basic test of generation of a flashrom fmap"""
2108         data = self._DoReadFile('094_fmap_x86.dts')
2109         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
2110         expected = U_BOOT_DATA + MRC_DATA + tools.get_bytes(ord('a'), 32 - 7)
2111         self.assertEqual(expected, data[:32])
2112         fhdr, fentries = fmap_util.DecodeFmap(data[32:])
2113
2114         self.assertEqual(0x100, fhdr.image_size)
2115
2116         self.assertEqual(0, fentries[0].offset)
2117         self.assertEqual(4, fentries[0].size)
2118         self.assertEqual(b'U_BOOT', fentries[0].name)
2119
2120         self.assertEqual(4, fentries[1].offset)
2121         self.assertEqual(3, fentries[1].size)
2122         self.assertEqual(b'INTEL_MRC', fentries[1].name)
2123
2124         self.assertEqual(32, fentries[2].offset)
2125         self.assertEqual(fmap_util.FMAP_HEADER_LEN +
2126                          fmap_util.FMAP_AREA_LEN * 3, fentries[2].size)
2127         self.assertEqual(b'FMAP', fentries[2].name)
2128
2129     def testFmapX86Section(self):
2130         """Basic test of generation of a flashrom fmap"""
2131         data = self._DoReadFile('095_fmap_x86_section.dts')
2132         expected = U_BOOT_DATA + MRC_DATA + tools.get_bytes(ord('b'), 32 - 7)
2133         self.assertEqual(expected, data[:32])
2134         fhdr, fentries = fmap_util.DecodeFmap(data[36:])
2135
2136         self.assertEqual(0x180, fhdr.image_size)
2137         expect_size = fmap_util.FMAP_HEADER_LEN + fmap_util.FMAP_AREA_LEN * 4
2138         fiter = iter(fentries)
2139
2140         fentry = next(fiter)
2141         self.assertEqual(b'U_BOOT', fentry.name)
2142         self.assertEqual(0, fentry.offset)
2143         self.assertEqual(4, fentry.size)
2144
2145         fentry = next(fiter)
2146         self.assertEqual(b'SECTION', fentry.name)
2147         self.assertEqual(4, fentry.offset)
2148         self.assertEqual(0x20 + expect_size, fentry.size)
2149
2150         fentry = next(fiter)
2151         self.assertEqual(b'INTEL_MRC', fentry.name)
2152         self.assertEqual(4, fentry.offset)
2153         self.assertEqual(3, fentry.size)
2154
2155         fentry = next(fiter)
2156         self.assertEqual(b'FMAP', fentry.name)
2157         self.assertEqual(36, fentry.offset)
2158         self.assertEqual(expect_size, fentry.size)
2159
2160     def testElf(self):
2161         """Basic test of ELF entries"""
2162         self._SetupSplElf()
2163         self._SetupTplElf()
2164         with open(self.ElfTestFile('bss_data'), 'rb') as fd:
2165             TestFunctional._MakeInputFile('-boot', fd.read())
2166         data = self._DoReadFile('096_elf.dts')
2167
2168     def testElfStrip(self):
2169         """Basic test of ELF entries"""
2170         self._SetupSplElf()
2171         with open(self.ElfTestFile('bss_data'), 'rb') as fd:
2172             TestFunctional._MakeInputFile('-boot', fd.read())
2173         data = self._DoReadFile('097_elf_strip.dts')
2174
2175     def testPackOverlapMap(self):
2176         """Test that overlapping regions are detected"""
2177         with test_util.capture_sys_output() as (stdout, stderr):
2178             with self.assertRaises(ValueError) as e:
2179                 self._DoTestFile('014_pack_overlap.dts', map=True)
2180         map_fname = tools.get_output_filename('image.map')
2181         self.assertEqual("Wrote map file '%s' to show errors\n" % map_fname,
2182                          stdout.getvalue())
2183
2184         # We should not get an inmage, but there should be a map file
2185         self.assertFalse(os.path.exists(tools.get_output_filename('image.bin')))
2186         self.assertTrue(os.path.exists(map_fname))
2187         map_data = tools.read_file(map_fname, binary=False)
2188         self.assertEqual('''ImagePos    Offset      Size  Name
2189 <none>    00000000  00000008  main-section
2190 <none>     00000000  00000004  u-boot
2191 <none>     00000003  00000004  u-boot-align
2192 ''', map_data)
2193
2194     def testPackRefCode(self):
2195         """Test that an image with an Intel Reference code binary works"""
2196         data = self._DoReadFile('100_intel_refcode.dts')
2197         self.assertEqual(REFCODE_DATA, data[:len(REFCODE_DATA)])
2198
2199     def testSectionOffset(self):
2200         """Tests use of a section with an offset"""
2201         data, _, map_data, _ = self._DoReadFileDtb('101_sections_offset.dts',
2202                                                    map=True)
2203         self.assertEqual('''ImagePos    Offset      Size  Name
2204 00000000  00000000  00000038  main-section
2205 00000004   00000004  00000010  section@0
2206 00000004    00000000  00000004  u-boot
2207 00000018   00000018  00000010  section@1
2208 00000018    00000000  00000004  u-boot
2209 0000002c   0000002c  00000004  section@2
2210 0000002c    00000000  00000004  u-boot
2211 ''', map_data)
2212         self.assertEqual(data,
2213                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2214                              tools.get_bytes(0x21, 12) +
2215                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2216                              tools.get_bytes(0x61, 12) +
2217                          tools.get_bytes(0x26, 4) + U_BOOT_DATA +
2218                              tools.get_bytes(0x26, 8))
2219
2220     def testCbfsRaw(self):
2221         """Test base handling of a Coreboot Filesystem (CBFS)
2222
2223         The exact contents of the CBFS is verified by similar tests in
2224         cbfs_util_test.py. The tests here merely check that the files added to
2225         the CBFS can be found in the final image.
2226         """
2227         data = self._DoReadFile('102_cbfs_raw.dts')
2228         size = 0xb0
2229
2230         cbfs = cbfs_util.CbfsReader(data)
2231         self.assertEqual(size, cbfs.rom_size)
2232
2233         self.assertIn('u-boot-dtb', cbfs.files)
2234         cfile = cbfs.files['u-boot-dtb']
2235         self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
2236
2237     def testCbfsArch(self):
2238         """Test on non-x86 architecture"""
2239         data = self._DoReadFile('103_cbfs_raw_ppc.dts')
2240         size = 0x100
2241
2242         cbfs = cbfs_util.CbfsReader(data)
2243         self.assertEqual(size, cbfs.rom_size)
2244
2245         self.assertIn('u-boot-dtb', cbfs.files)
2246         cfile = cbfs.files['u-boot-dtb']
2247         self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
2248
2249     def testCbfsStage(self):
2250         """Tests handling of a Coreboot Filesystem (CBFS)"""
2251         if not elf.ELF_TOOLS:
2252             self.skipTest('Python elftools not available')
2253         elf_fname = os.path.join(self._indir, 'cbfs-stage.elf')
2254         elf.MakeElf(elf_fname, U_BOOT_DATA, U_BOOT_DTB_DATA)
2255         size = 0xb0
2256
2257         data = self._DoReadFile('104_cbfs_stage.dts')
2258         cbfs = cbfs_util.CbfsReader(data)
2259         self.assertEqual(size, cbfs.rom_size)
2260
2261         self.assertIn('u-boot', cbfs.files)
2262         cfile = cbfs.files['u-boot']
2263         self.assertEqual(U_BOOT_DATA + U_BOOT_DTB_DATA, cfile.data)
2264
2265     def testCbfsRawCompress(self):
2266         """Test handling of compressing raw files"""
2267         self._CheckLz4()
2268         data = self._DoReadFile('105_cbfs_raw_compress.dts')
2269         size = 0x140
2270
2271         cbfs = cbfs_util.CbfsReader(data)
2272         self.assertIn('u-boot', cbfs.files)
2273         cfile = cbfs.files['u-boot']
2274         self.assertEqual(COMPRESS_DATA, cfile.data)
2275
2276     def testCbfsBadArch(self):
2277         """Test handling of a bad architecture"""
2278         with self.assertRaises(ValueError) as e:
2279             self._DoReadFile('106_cbfs_bad_arch.dts')
2280         self.assertIn("Invalid architecture 'bad-arch'", str(e.exception))
2281
2282     def testCbfsNoSize(self):
2283         """Test handling of a missing size property"""
2284         with self.assertRaises(ValueError) as e:
2285             self._DoReadFile('107_cbfs_no_size.dts')
2286         self.assertIn('entry must have a size property', str(e.exception))
2287
2288     def testCbfsNoContents(self):
2289         """Test handling of a CBFS entry which does not provide contentsy"""
2290         with self.assertRaises(ValueError) as e:
2291             self._DoReadFile('108_cbfs_no_contents.dts')
2292         self.assertIn('Could not complete processing of contents',
2293                       str(e.exception))
2294
2295     def testCbfsBadCompress(self):
2296         """Test handling of a bad architecture"""
2297         with self.assertRaises(ValueError) as e:
2298             self._DoReadFile('109_cbfs_bad_compress.dts')
2299         self.assertIn("Invalid compression in 'u-boot': 'invalid-algo'",
2300                       str(e.exception))
2301
2302     def testCbfsNamedEntries(self):
2303         """Test handling of named entries"""
2304         data = self._DoReadFile('110_cbfs_name.dts')
2305
2306         cbfs = cbfs_util.CbfsReader(data)
2307         self.assertIn('FRED', cbfs.files)
2308         cfile1 = cbfs.files['FRED']
2309         self.assertEqual(U_BOOT_DATA, cfile1.data)
2310
2311         self.assertIn('hello', cbfs.files)
2312         cfile2 = cbfs.files['hello']
2313         self.assertEqual(U_BOOT_DTB_DATA, cfile2.data)
2314
2315     def _SetupIfwi(self, fname):
2316         """Set up to run an IFWI test
2317
2318         Args:
2319             fname: Filename of input file to provide (fitimage.bin or ifwi.bin)
2320         """
2321         self._SetupSplElf()
2322         self._SetupTplElf()
2323
2324         # Intel Integrated Firmware Image (IFWI) file
2325         with gzip.open(self.TestFile('%s.gz' % fname), 'rb') as fd:
2326             data = fd.read()
2327         TestFunctional._MakeInputFile(fname,data)
2328
2329     def _CheckIfwi(self, data):
2330         """Check that an image with an IFWI contains the correct output
2331
2332         Args:
2333             data: Conents of output file
2334         """
2335         expected_desc = tools.read_file(self.TestFile('descriptor.bin'))
2336         if data[:0x1000] != expected_desc:
2337             self.fail('Expected descriptor binary at start of image')
2338
2339         # We expect to find the TPL wil in subpart IBBP entry IBBL
2340         image_fname = tools.get_output_filename('image.bin')
2341         tpl_fname = tools.get_output_filename('tpl.out')
2342         ifwitool = bintool.Bintool.create('ifwitool')
2343         ifwitool.extract(image_fname, 'IBBP', 'IBBL', tpl_fname)
2344
2345         tpl_data = tools.read_file(tpl_fname)
2346         self.assertEqual(U_BOOT_TPL_DATA, tpl_data[:len(U_BOOT_TPL_DATA)])
2347
2348     def testPackX86RomIfwi(self):
2349         """Test that an x86 ROM with Integrated Firmware Image can be created"""
2350         self._SetupIfwi('fitimage.bin')
2351         data = self._DoReadFile('111_x86_rom_ifwi.dts')
2352         self._CheckIfwi(data)
2353
2354     def testPackX86RomIfwiNoDesc(self):
2355         """Test that an x86 ROM with IFWI can be created from an ifwi.bin file"""
2356         self._SetupIfwi('ifwi.bin')
2357         data = self._DoReadFile('112_x86_rom_ifwi_nodesc.dts')
2358         self._CheckIfwi(data)
2359
2360     def testPackX86RomIfwiNoData(self):
2361         """Test that an x86 ROM with IFWI handles missing data"""
2362         self._SetupIfwi('ifwi.bin')
2363         with self.assertRaises(ValueError) as e:
2364             data = self._DoReadFile('113_x86_rom_ifwi_nodata.dts')
2365         self.assertIn('Could not complete processing of contents',
2366                       str(e.exception))
2367
2368     def testIfwiMissing(self):
2369         """Test that binman still produces an image if ifwitool is missing"""
2370         self._SetupIfwi('fitimage.bin')
2371         with test_util.capture_sys_output() as (_, stderr):
2372             self._DoTestFile('111_x86_rom_ifwi.dts',
2373                              force_missing_bintools='ifwitool')
2374         err = stderr.getvalue()
2375         self.assertRegex(err,
2376                          "Image 'main-section'.*missing bintools.*: ifwitool")
2377
2378     def testCbfsOffset(self):
2379         """Test a CBFS with files at particular offsets
2380
2381         Like all CFBS tests, this is just checking the logic that calls
2382         cbfs_util. See cbfs_util_test for fully tests (e.g. test_cbfs_offset()).
2383         """
2384         data = self._DoReadFile('114_cbfs_offset.dts')
2385         size = 0x200
2386
2387         cbfs = cbfs_util.CbfsReader(data)
2388         self.assertEqual(size, cbfs.rom_size)
2389
2390         self.assertIn('u-boot', cbfs.files)
2391         cfile = cbfs.files['u-boot']
2392         self.assertEqual(U_BOOT_DATA, cfile.data)
2393         self.assertEqual(0x40, cfile.cbfs_offset)
2394
2395         self.assertIn('u-boot-dtb', cbfs.files)
2396         cfile2 = cbfs.files['u-boot-dtb']
2397         self.assertEqual(U_BOOT_DTB_DATA, cfile2.data)
2398         self.assertEqual(0x140, cfile2.cbfs_offset)
2399
2400     def testFdtmap(self):
2401         """Test an FDT map can be inserted in the image"""
2402         data = self.data = self._DoReadFileRealDtb('115_fdtmap.dts')
2403         fdtmap_data = data[len(U_BOOT_DATA):]
2404         magic = fdtmap_data[:8]
2405         self.assertEqual(b'_FDTMAP_', magic)
2406         self.assertEqual(tools.get_bytes(0, 8), fdtmap_data[8:16])
2407
2408         fdt_data = fdtmap_data[16:]
2409         dtb = fdt.Fdt.FromData(fdt_data)
2410         dtb.Scan()
2411         props = self._GetPropTree(dtb, BASE_DTB_PROPS, prefix='/')
2412         self.assertEqual({
2413             'image-pos': 0,
2414             'offset': 0,
2415             'u-boot:offset': 0,
2416             'u-boot:size': len(U_BOOT_DATA),
2417             'u-boot:image-pos': 0,
2418             'fdtmap:image-pos': 4,
2419             'fdtmap:offset': 4,
2420             'fdtmap:size': len(fdtmap_data),
2421             'size': len(data),
2422         }, props)
2423
2424     def testFdtmapNoMatch(self):
2425         """Check handling of an FDT map when the section cannot be found"""
2426         self.data = self._DoReadFileRealDtb('115_fdtmap.dts')
2427
2428         # Mangle the section name, which should cause a mismatch between the
2429         # correct FDT path and the one expected by the section
2430         image = control.images['image']
2431         image._node.path += '-suffix'
2432         entries = image.GetEntries()
2433         fdtmap = entries['fdtmap']
2434         with self.assertRaises(ValueError) as e:
2435             fdtmap._GetFdtmap()
2436         self.assertIn("Cannot locate node for path '/binman-suffix'",
2437                       str(e.exception))
2438
2439     def testFdtmapHeader(self):
2440         """Test an FDT map and image header can be inserted in the image"""
2441         data = self.data = self._DoReadFileRealDtb('116_fdtmap_hdr.dts')
2442         fdtmap_pos = len(U_BOOT_DATA)
2443         fdtmap_data = data[fdtmap_pos:]
2444         fdt_data = fdtmap_data[16:]
2445         dtb = fdt.Fdt.FromData(fdt_data)
2446         fdt_size = dtb.GetFdtObj().totalsize()
2447         hdr_data = data[-8:]
2448         self.assertEqual(b'BinM', hdr_data[:4])
2449         offset = struct.unpack('<I', hdr_data[4:])[0] & 0xffffffff
2450         self.assertEqual(fdtmap_pos - 0x400, offset - (1 << 32))
2451
2452     def testFdtmapHeaderStart(self):
2453         """Test an image header can be inserted at the image start"""
2454         data = self.data = self._DoReadFileRealDtb('117_fdtmap_hdr_start.dts')
2455         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
2456         hdr_data = data[:8]
2457         self.assertEqual(b'BinM', hdr_data[:4])
2458         offset = struct.unpack('<I', hdr_data[4:])[0]
2459         self.assertEqual(fdtmap_pos, offset)
2460
2461     def testFdtmapHeaderPos(self):
2462         """Test an image header can be inserted at a chosen position"""
2463         data = self.data = self._DoReadFileRealDtb('118_fdtmap_hdr_pos.dts')
2464         fdtmap_pos = 0x100 + len(U_BOOT_DATA)
2465         hdr_data = data[0x80:0x88]
2466         self.assertEqual(b'BinM', hdr_data[:4])
2467         offset = struct.unpack('<I', hdr_data[4:])[0]
2468         self.assertEqual(fdtmap_pos, offset)
2469
2470     def testHeaderMissingFdtmap(self):
2471         """Test an image header requires an fdtmap"""
2472         with self.assertRaises(ValueError) as e:
2473             self.data = self._DoReadFileRealDtb('119_fdtmap_hdr_missing.dts')
2474         self.assertIn("'image_header' section must have an 'fdtmap' sibling",
2475                       str(e.exception))
2476
2477     def testHeaderNoLocation(self):
2478         """Test an image header with a no specified location is detected"""
2479         with self.assertRaises(ValueError) as e:
2480             self.data = self._DoReadFileRealDtb('120_hdr_no_location.dts')
2481         self.assertIn("Invalid location 'None', expected 'start' or 'end'",
2482                       str(e.exception))
2483
2484     def testEntryExpand(self):
2485         """Test expanding an entry after it is packed"""
2486         data = self._DoReadFile('121_entry_expand.dts')
2487         self.assertEqual(b'aaa', data[:3])
2488         self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)])
2489         self.assertEqual(b'aaa', data[-3:])
2490
2491     def testEntryExpandBad(self):
2492         """Test expanding an entry after it is packed, twice"""
2493         with self.assertRaises(ValueError) as e:
2494             self._DoReadFile('122_entry_expand_twice.dts')
2495         self.assertIn("Image '/binman': Entries changed size after packing",
2496                       str(e.exception))
2497
2498     def testEntryExpandSection(self):
2499         """Test expanding an entry within a section after it is packed"""
2500         data = self._DoReadFile('123_entry_expand_section.dts')
2501         self.assertEqual(b'aaa', data[:3])
2502         self.assertEqual(U_BOOT_DATA, data[3:3 + len(U_BOOT_DATA)])
2503         self.assertEqual(b'aaa', data[-3:])
2504
2505     def testCompressDtb(self):
2506         """Test that compress of device-tree files is supported"""
2507         self._CheckLz4()
2508         data = self.data = self._DoReadFileRealDtb('124_compress_dtb.dts')
2509         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
2510         comp_data = data[len(U_BOOT_DATA):]
2511         orig = self._decompress(comp_data)
2512         dtb = fdt.Fdt.FromData(orig)
2513         dtb.Scan()
2514         props = self._GetPropTree(dtb, ['size', 'uncomp-size'])
2515         expected = {
2516             'u-boot:size': len(U_BOOT_DATA),
2517             'u-boot-dtb:uncomp-size': len(orig),
2518             'u-boot-dtb:size': len(comp_data),
2519             'size': len(data),
2520             }
2521         self.assertEqual(expected, props)
2522
2523     def testCbfsUpdateFdt(self):
2524         """Test that we can update the device tree with CBFS offset/size info"""
2525         self._CheckLz4()
2526         data, _, _, out_dtb_fname = self._DoReadFileDtb('125_cbfs_update.dts',
2527                                                         update_dtb=True)
2528         dtb = fdt.Fdt(out_dtb_fname)
2529         dtb.Scan()
2530         props = self._GetPropTree(dtb, BASE_DTB_PROPS + ['uncomp-size'])
2531         del props['cbfs/u-boot:size']
2532         self.assertEqual({
2533             'offset': 0,
2534             'size': len(data),
2535             'image-pos': 0,
2536             'cbfs:offset': 0,
2537             'cbfs:size': len(data),
2538             'cbfs:image-pos': 0,
2539             'cbfs/u-boot:offset': 0x38,
2540             'cbfs/u-boot:uncomp-size': len(U_BOOT_DATA),
2541             'cbfs/u-boot:image-pos': 0x38,
2542             'cbfs/u-boot-dtb:offset': 0xb8,
2543             'cbfs/u-boot-dtb:size': len(U_BOOT_DATA),
2544             'cbfs/u-boot-dtb:image-pos': 0xb8,
2545             }, props)
2546
2547     def testCbfsBadType(self):
2548         """Test an image header with a no specified location is detected"""
2549         with self.assertRaises(ValueError) as e:
2550             self._DoReadFile('126_cbfs_bad_type.dts')
2551         self.assertIn("Unknown cbfs-type 'badtype'", str(e.exception))
2552
2553     def testList(self):
2554         """Test listing the files in an image"""
2555         self._CheckLz4()
2556         data = self._DoReadFile('127_list.dts')
2557         image = control.images['image']
2558         entries = image.BuildEntryList()
2559         self.assertEqual(7, len(entries))
2560
2561         ent = entries[0]
2562         self.assertEqual(0, ent.indent)
2563         self.assertEqual('main-section', ent.name)
2564         self.assertEqual('section', ent.etype)
2565         self.assertEqual(len(data), ent.size)
2566         self.assertEqual(0, ent.image_pos)
2567         self.assertEqual(None, ent.uncomp_size)
2568         self.assertEqual(0, ent.offset)
2569
2570         ent = entries[1]
2571         self.assertEqual(1, ent.indent)
2572         self.assertEqual('u-boot', ent.name)
2573         self.assertEqual('u-boot', ent.etype)
2574         self.assertEqual(len(U_BOOT_DATA), ent.size)
2575         self.assertEqual(0, ent.image_pos)
2576         self.assertEqual(None, ent.uncomp_size)
2577         self.assertEqual(0, ent.offset)
2578
2579         ent = entries[2]
2580         self.assertEqual(1, ent.indent)
2581         self.assertEqual('section', ent.name)
2582         self.assertEqual('section', ent.etype)
2583         section_size = ent.size
2584         self.assertEqual(0x100, ent.image_pos)
2585         self.assertEqual(None, ent.uncomp_size)
2586         self.assertEqual(0x100, ent.offset)
2587
2588         ent = entries[3]
2589         self.assertEqual(2, ent.indent)
2590         self.assertEqual('cbfs', ent.name)
2591         self.assertEqual('cbfs', ent.etype)
2592         self.assertEqual(0x400, ent.size)
2593         self.assertEqual(0x100, ent.image_pos)
2594         self.assertEqual(None, ent.uncomp_size)
2595         self.assertEqual(0, ent.offset)
2596
2597         ent = entries[4]
2598         self.assertEqual(3, ent.indent)
2599         self.assertEqual('u-boot', ent.name)
2600         self.assertEqual('u-boot', ent.etype)
2601         self.assertEqual(len(U_BOOT_DATA), ent.size)
2602         self.assertEqual(0x138, ent.image_pos)
2603         self.assertEqual(None, ent.uncomp_size)
2604         self.assertEqual(0x38, ent.offset)
2605
2606         ent = entries[5]
2607         self.assertEqual(3, ent.indent)
2608         self.assertEqual('u-boot-dtb', ent.name)
2609         self.assertEqual('text', ent.etype)
2610         self.assertGreater(len(COMPRESS_DATA), ent.size)
2611         self.assertEqual(0x178, ent.image_pos)
2612         self.assertEqual(len(COMPRESS_DATA), ent.uncomp_size)
2613         self.assertEqual(0x78, ent.offset)
2614
2615         ent = entries[6]
2616         self.assertEqual(2, ent.indent)
2617         self.assertEqual('u-boot-dtb', ent.name)
2618         self.assertEqual('u-boot-dtb', ent.etype)
2619         self.assertEqual(0x500, ent.image_pos)
2620         self.assertEqual(len(U_BOOT_DTB_DATA), ent.uncomp_size)
2621         dtb_size = ent.size
2622         # Compressing this data expands it since headers are added
2623         self.assertGreater(dtb_size, len(U_BOOT_DTB_DATA))
2624         self.assertEqual(0x400, ent.offset)
2625
2626         self.assertEqual(len(data), 0x100 + section_size)
2627         self.assertEqual(section_size, 0x400 + dtb_size)
2628
2629     def testFindFdtmap(self):
2630         """Test locating an FDT map in an image"""
2631         self._CheckLz4()
2632         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2633         image = control.images['image']
2634         entries = image.GetEntries()
2635         entry = entries['fdtmap']
2636         self.assertEqual(entry.image_pos, fdtmap.LocateFdtmap(data))
2637
2638     def testFindFdtmapMissing(self):
2639         """Test failing to locate an FDP map"""
2640         data = self._DoReadFile('005_simple.dts')
2641         self.assertEqual(None, fdtmap.LocateFdtmap(data))
2642
2643     def testFindImageHeader(self):
2644         """Test locating a image header"""
2645         self._CheckLz4()
2646         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2647         image = control.images['image']
2648         entries = image.GetEntries()
2649         entry = entries['fdtmap']
2650         # The header should point to the FDT map
2651         self.assertEqual(entry.image_pos, image_header.LocateHeaderOffset(data))
2652
2653     def testFindImageHeaderStart(self):
2654         """Test locating a image header located at the start of an image"""
2655         data = self.data = self._DoReadFileRealDtb('117_fdtmap_hdr_start.dts')
2656         image = control.images['image']
2657         entries = image.GetEntries()
2658         entry = entries['fdtmap']
2659         # The header should point to the FDT map
2660         self.assertEqual(entry.image_pos, image_header.LocateHeaderOffset(data))
2661
2662     def testFindImageHeaderMissing(self):
2663         """Test failing to locate an image header"""
2664         data = self._DoReadFile('005_simple.dts')
2665         self.assertEqual(None, image_header.LocateHeaderOffset(data))
2666
2667     def testReadImage(self):
2668         """Test reading an image and accessing its FDT map"""
2669         self._CheckLz4()
2670         data = self.data = self._DoReadFileRealDtb('128_decode_image.dts')
2671         image_fname = tools.get_output_filename('image.bin')
2672         orig_image = control.images['image']
2673         image = Image.FromFile(image_fname)
2674         self.assertEqual(orig_image.GetEntries().keys(),
2675                          image.GetEntries().keys())
2676
2677         orig_entry = orig_image.GetEntries()['fdtmap']
2678         entry = image.GetEntries()['fdtmap']
2679         self.assertEquals(orig_entry.offset, entry.offset)
2680         self.assertEquals(orig_entry.size, entry.size)
2681         self.assertEquals(orig_entry.image_pos, entry.image_pos)
2682
2683     def testReadImageNoHeader(self):
2684         """Test accessing an image's FDT map without an image header"""
2685         self._CheckLz4()
2686         data = self._DoReadFileRealDtb('129_decode_image_nohdr.dts')
2687         image_fname = tools.get_output_filename('image.bin')
2688         image = Image.FromFile(image_fname)
2689         self.assertTrue(isinstance(image, Image))
2690         self.assertEqual('image', image.image_name[-5:])
2691
2692     def testReadImageFail(self):
2693         """Test failing to read an image image's FDT map"""
2694         self._DoReadFile('005_simple.dts')
2695         image_fname = tools.get_output_filename('image.bin')
2696         with self.assertRaises(ValueError) as e:
2697             image = Image.FromFile(image_fname)
2698         self.assertIn("Cannot find FDT map in image", str(e.exception))
2699
2700     def testListCmd(self):
2701         """Test listing the files in an image using an Fdtmap"""
2702         self._CheckLz4()
2703         data = self._DoReadFileRealDtb('130_list_fdtmap.dts')
2704
2705         # lz4 compression size differs depending on the version
2706         image = control.images['image']
2707         entries = image.GetEntries()
2708         section_size = entries['section'].size
2709         fdt_size = entries['section'].GetEntries()['u-boot-dtb'].size
2710         fdtmap_offset = entries['fdtmap'].offset
2711
2712         try:
2713             tmpdir, updated_fname = self._SetupImageInTmpdir()
2714             with test_util.capture_sys_output() as (stdout, stderr):
2715                 self._DoBinman('ls', '-i', updated_fname)
2716         finally:
2717             shutil.rmtree(tmpdir)
2718         lines = stdout.getvalue().splitlines()
2719         expected = [
2720 'Name              Image-pos  Size  Entry-type    Offset  Uncomp-size',
2721 '----------------------------------------------------------------------',
2722 'main-section              0   c00  section            0',
2723 '  u-boot                  0     4  u-boot             0',
2724 '  section               100   %x  section          100' % section_size,
2725 '    cbfs                100   400  cbfs               0',
2726 '      u-boot            138     4  u-boot            38',
2727 '      u-boot-dtb        180   105  u-boot-dtb        80          3c9',
2728 '    u-boot-dtb          500   %x  u-boot-dtb       400          3c9' % fdt_size,
2729 '  fdtmap                %x   3bd  fdtmap           %x' %
2730         (fdtmap_offset, fdtmap_offset),
2731 '  image-header          bf8     8  image-header     bf8',
2732             ]
2733         self.assertEqual(expected, lines)
2734
2735     def testListCmdFail(self):
2736         """Test failing to list an image"""
2737         self._DoReadFile('005_simple.dts')
2738         try:
2739             tmpdir, updated_fname = self._SetupImageInTmpdir()
2740             with self.assertRaises(ValueError) as e:
2741                 self._DoBinman('ls', '-i', updated_fname)
2742         finally:
2743             shutil.rmtree(tmpdir)
2744         self.assertIn("Cannot find FDT map in image", str(e.exception))
2745
2746     def _RunListCmd(self, paths, expected):
2747         """List out entries and check the result
2748
2749         Args:
2750             paths: List of paths to pass to the list command
2751             expected: Expected list of filenames to be returned, in order
2752         """
2753         self._CheckLz4()
2754         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2755         image_fname = tools.get_output_filename('image.bin')
2756         image = Image.FromFile(image_fname)
2757         lines = image.GetListEntries(paths)[1]
2758         files = [line[0].strip() for line in lines[1:]]
2759         self.assertEqual(expected, files)
2760
2761     def testListCmdSection(self):
2762         """Test listing the files in a section"""
2763         self._RunListCmd(['section'],
2764             ['section', 'cbfs', 'u-boot', 'u-boot-dtb', 'u-boot-dtb'])
2765
2766     def testListCmdFile(self):
2767         """Test listing a particular file"""
2768         self._RunListCmd(['*u-boot-dtb'], ['u-boot-dtb', 'u-boot-dtb'])
2769
2770     def testListCmdWildcard(self):
2771         """Test listing a wildcarded file"""
2772         self._RunListCmd(['*boot*'],
2773             ['u-boot', 'u-boot', 'u-boot-dtb', 'u-boot-dtb'])
2774
2775     def testListCmdWildcardMulti(self):
2776         """Test listing a wildcarded file"""
2777         self._RunListCmd(['*cb*', '*head*'],
2778             ['cbfs', 'u-boot', 'u-boot-dtb', 'image-header'])
2779
2780     def testListCmdEmpty(self):
2781         """Test listing a wildcarded file"""
2782         self._RunListCmd(['nothing'], [])
2783
2784     def testListCmdPath(self):
2785         """Test listing the files in a sub-entry of a section"""
2786         self._RunListCmd(['section/cbfs'], ['cbfs', 'u-boot', 'u-boot-dtb'])
2787
2788     def _RunExtractCmd(self, entry_name, decomp=True):
2789         """Extract an entry from an image
2790
2791         Args:
2792             entry_name: Entry name to extract
2793             decomp: True to decompress the data if compressed, False to leave
2794                 it in its raw uncompressed format
2795
2796         Returns:
2797             data from entry
2798         """
2799         self._CheckLz4()
2800         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2801         image_fname = tools.get_output_filename('image.bin')
2802         return control.ReadEntry(image_fname, entry_name, decomp)
2803
2804     def testExtractSimple(self):
2805         """Test extracting a single file"""
2806         data = self._RunExtractCmd('u-boot')
2807         self.assertEqual(U_BOOT_DATA, data)
2808
2809     def testExtractSection(self):
2810         """Test extracting the files in a section"""
2811         data = self._RunExtractCmd('section')
2812         cbfs_data = data[:0x400]
2813         cbfs = cbfs_util.CbfsReader(cbfs_data)
2814         self.assertEqual(['u-boot', 'u-boot-dtb', ''], list(cbfs.files.keys()))
2815         dtb_data = data[0x400:]
2816         dtb = self._decompress(dtb_data)
2817         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2818
2819     def testExtractCompressed(self):
2820         """Test extracting compressed data"""
2821         data = self._RunExtractCmd('section/u-boot-dtb')
2822         self.assertEqual(EXTRACT_DTB_SIZE, len(data))
2823
2824     def testExtractRaw(self):
2825         """Test extracting compressed data without decompressing it"""
2826         data = self._RunExtractCmd('section/u-boot-dtb', decomp=False)
2827         dtb = self._decompress(data)
2828         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2829
2830     def testExtractCbfs(self):
2831         """Test extracting CBFS data"""
2832         data = self._RunExtractCmd('section/cbfs/u-boot')
2833         self.assertEqual(U_BOOT_DATA, data)
2834
2835     def testExtractCbfsCompressed(self):
2836         """Test extracting CBFS compressed data"""
2837         data = self._RunExtractCmd('section/cbfs/u-boot-dtb')
2838         self.assertEqual(EXTRACT_DTB_SIZE, len(data))
2839
2840     def testExtractCbfsRaw(self):
2841         """Test extracting CBFS compressed data without decompressing it"""
2842         data = self._RunExtractCmd('section/cbfs/u-boot-dtb', decomp=False)
2843         dtb = comp_util.decompress(data, 'lzma', with_header=False)
2844         self.assertEqual(EXTRACT_DTB_SIZE, len(dtb))
2845
2846     def testExtractBadEntry(self):
2847         """Test extracting a bad section path"""
2848         with self.assertRaises(ValueError) as e:
2849             self._RunExtractCmd('section/does-not-exist')
2850         self.assertIn("Entry 'does-not-exist' not found in '/section'",
2851                       str(e.exception))
2852
2853     def testExtractMissingFile(self):
2854         """Test extracting file that does not exist"""
2855         with self.assertRaises(IOError) as e:
2856             control.ReadEntry('missing-file', 'name')
2857
2858     def testExtractBadFile(self):
2859         """Test extracting an invalid file"""
2860         fname = os.path.join(self._indir, 'badfile')
2861         tools.write_file(fname, b'')
2862         with self.assertRaises(ValueError) as e:
2863             control.ReadEntry(fname, 'name')
2864
2865     def testExtractCmd(self):
2866         """Test extracting a file fron an image on the command line"""
2867         self._CheckLz4()
2868         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2869         fname = os.path.join(self._indir, 'output.extact')
2870         try:
2871             tmpdir, updated_fname = self._SetupImageInTmpdir()
2872             with test_util.capture_sys_output() as (stdout, stderr):
2873                 self._DoBinman('extract', '-i', updated_fname, 'u-boot',
2874                                '-f', fname)
2875         finally:
2876             shutil.rmtree(tmpdir)
2877         data = tools.read_file(fname)
2878         self.assertEqual(U_BOOT_DATA, data)
2879
2880     def testExtractOneEntry(self):
2881         """Test extracting a single entry fron an image """
2882         self._CheckLz4()
2883         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2884         image_fname = tools.get_output_filename('image.bin')
2885         fname = os.path.join(self._indir, 'output.extact')
2886         control.ExtractEntries(image_fname, fname, None, ['u-boot'])
2887         data = tools.read_file(fname)
2888         self.assertEqual(U_BOOT_DATA, data)
2889
2890     def _CheckExtractOutput(self, decomp):
2891         """Helper to test file output with and without decompression
2892
2893         Args:
2894             decomp: True to decompress entry data, False to output it raw
2895         """
2896         def _CheckPresent(entry_path, expect_data, expect_size=None):
2897             """Check and remove expected file
2898
2899             This checks the data/size of a file and removes the file both from
2900             the outfiles set and from the output directory. Once all files are
2901             processed, both the set and directory should be empty.
2902
2903             Args:
2904                 entry_path: Entry path
2905                 expect_data: Data to expect in file, or None to skip check
2906                 expect_size: Size of data to expect in file, or None to skip
2907             """
2908             path = os.path.join(outdir, entry_path)
2909             data = tools.read_file(path)
2910             os.remove(path)
2911             if expect_data:
2912                 self.assertEqual(expect_data, data)
2913             elif expect_size:
2914                 self.assertEqual(expect_size, len(data))
2915             outfiles.remove(path)
2916
2917         def _CheckDirPresent(name):
2918             """Remove expected directory
2919
2920             This gives an error if the directory does not exist as expected
2921
2922             Args:
2923                 name: Name of directory to remove
2924             """
2925             path = os.path.join(outdir, name)
2926             os.rmdir(path)
2927
2928         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2929         image_fname = tools.get_output_filename('image.bin')
2930         outdir = os.path.join(self._indir, 'extract')
2931         einfos = control.ExtractEntries(image_fname, None, outdir, [], decomp)
2932
2933         # Create a set of all file that were output (should be 9)
2934         outfiles = set()
2935         for root, dirs, files in os.walk(outdir):
2936             outfiles |= set([os.path.join(root, fname) for fname in files])
2937         self.assertEqual(9, len(outfiles))
2938         self.assertEqual(9, len(einfos))
2939
2940         image = control.images['image']
2941         entries = image.GetEntries()
2942
2943         # Check the 9 files in various ways
2944         section = entries['section']
2945         section_entries = section.GetEntries()
2946         cbfs_entries = section_entries['cbfs'].GetEntries()
2947         _CheckPresent('u-boot', U_BOOT_DATA)
2948         _CheckPresent('section/cbfs/u-boot', U_BOOT_DATA)
2949         dtb_len = EXTRACT_DTB_SIZE
2950         if not decomp:
2951             dtb_len = cbfs_entries['u-boot-dtb'].size
2952         _CheckPresent('section/cbfs/u-boot-dtb', None, dtb_len)
2953         if not decomp:
2954             dtb_len = section_entries['u-boot-dtb'].size
2955         _CheckPresent('section/u-boot-dtb', None, dtb_len)
2956
2957         fdtmap = entries['fdtmap']
2958         _CheckPresent('fdtmap', fdtmap.data)
2959         hdr = entries['image-header']
2960         _CheckPresent('image-header', hdr.data)
2961
2962         _CheckPresent('section/root', section.data)
2963         cbfs = section_entries['cbfs']
2964         _CheckPresent('section/cbfs/root', cbfs.data)
2965         data = tools.read_file(image_fname)
2966         _CheckPresent('root', data)
2967
2968         # There should be no files left. Remove all the directories to check.
2969         # If there are any files/dirs remaining, one of these checks will fail.
2970         self.assertEqual(0, len(outfiles))
2971         _CheckDirPresent('section/cbfs')
2972         _CheckDirPresent('section')
2973         _CheckDirPresent('')
2974         self.assertFalse(os.path.exists(outdir))
2975
2976     def testExtractAllEntries(self):
2977         """Test extracting all entries"""
2978         self._CheckLz4()
2979         self._CheckExtractOutput(decomp=True)
2980
2981     def testExtractAllEntriesRaw(self):
2982         """Test extracting all entries without decompressing them"""
2983         self._CheckLz4()
2984         self._CheckExtractOutput(decomp=False)
2985
2986     def testExtractSelectedEntries(self):
2987         """Test extracting some entries"""
2988         self._CheckLz4()
2989         self._DoReadFileRealDtb('130_list_fdtmap.dts')
2990         image_fname = tools.get_output_filename('image.bin')
2991         outdir = os.path.join(self._indir, 'extract')
2992         einfos = control.ExtractEntries(image_fname, None, outdir,
2993                                         ['*cb*', '*head*'])
2994
2995         # File output is tested by testExtractAllEntries(), so just check that
2996         # the expected entries are selected
2997         names = [einfo.name for einfo in einfos]
2998         self.assertEqual(names,
2999                          ['cbfs', 'u-boot', 'u-boot-dtb', 'image-header'])
3000
3001     def testExtractNoEntryPaths(self):
3002         """Test extracting some entries"""
3003         self._CheckLz4()
3004         self._DoReadFileRealDtb('130_list_fdtmap.dts')
3005         image_fname = tools.get_output_filename('image.bin')
3006         with self.assertRaises(ValueError) as e:
3007             control.ExtractEntries(image_fname, 'fname', None, [])
3008         self.assertIn('Must specify an entry path to write with -f',
3009                       str(e.exception))
3010
3011     def testExtractTooManyEntryPaths(self):
3012         """Test extracting some entries"""
3013         self._CheckLz4()
3014         self._DoReadFileRealDtb('130_list_fdtmap.dts')
3015         image_fname = tools.get_output_filename('image.bin')
3016         with self.assertRaises(ValueError) as e:
3017             control.ExtractEntries(image_fname, 'fname', None, ['a', 'b'])
3018         self.assertIn('Must specify exactly one entry path to write with -f',
3019                       str(e.exception))
3020
3021     def testPackAlignSection(self):
3022         """Test that sections can have alignment"""
3023         self._DoReadFile('131_pack_align_section.dts')
3024
3025         self.assertIn('image', control.images)
3026         image = control.images['image']
3027         entries = image.GetEntries()
3028         self.assertEqual(3, len(entries))
3029
3030         # First u-boot
3031         self.assertIn('u-boot', entries)
3032         entry = entries['u-boot']
3033         self.assertEqual(0, entry.offset)
3034         self.assertEqual(0, entry.image_pos)
3035         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3036         self.assertEqual(len(U_BOOT_DATA), entry.size)
3037
3038         # Section0
3039         self.assertIn('section0', entries)
3040         section0 = entries['section0']
3041         self.assertEqual(0x10, section0.offset)
3042         self.assertEqual(0x10, section0.image_pos)
3043         self.assertEqual(len(U_BOOT_DATA), section0.size)
3044
3045         # Second u-boot
3046         section_entries = section0.GetEntries()
3047         self.assertIn('u-boot', section_entries)
3048         entry = section_entries['u-boot']
3049         self.assertEqual(0, entry.offset)
3050         self.assertEqual(0x10, entry.image_pos)
3051         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3052         self.assertEqual(len(U_BOOT_DATA), entry.size)
3053
3054         # Section1
3055         self.assertIn('section1', entries)
3056         section1 = entries['section1']
3057         self.assertEqual(0x14, section1.offset)
3058         self.assertEqual(0x14, section1.image_pos)
3059         self.assertEqual(0x20, section1.size)
3060
3061         # Second u-boot
3062         section_entries = section1.GetEntries()
3063         self.assertIn('u-boot', section_entries)
3064         entry = section_entries['u-boot']
3065         self.assertEqual(0, entry.offset)
3066         self.assertEqual(0x14, entry.image_pos)
3067         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3068         self.assertEqual(len(U_BOOT_DATA), entry.size)
3069
3070         # Section2
3071         self.assertIn('section2', section_entries)
3072         section2 = section_entries['section2']
3073         self.assertEqual(0x4, section2.offset)
3074         self.assertEqual(0x18, section2.image_pos)
3075         self.assertEqual(4, section2.size)
3076
3077         # Third u-boot
3078         section_entries = section2.GetEntries()
3079         self.assertIn('u-boot', section_entries)
3080         entry = section_entries['u-boot']
3081         self.assertEqual(0, entry.offset)
3082         self.assertEqual(0x18, entry.image_pos)
3083         self.assertEqual(len(U_BOOT_DATA), entry.contents_size)
3084         self.assertEqual(len(U_BOOT_DATA), entry.size)
3085
3086     def _RunReplaceCmd(self, entry_name, data, decomp=True, allow_resize=True,
3087                        dts='132_replace.dts'):
3088         """Replace an entry in an image
3089
3090         This writes the entry data to update it, then opens the updated file and
3091         returns the value that it now finds there.
3092
3093         Args:
3094             entry_name: Entry name to replace
3095             data: Data to replace it with
3096             decomp: True to compress the data if needed, False if data is
3097                 already compressed so should be used as is
3098             allow_resize: True to allow entries to change size, False to raise
3099                 an exception
3100
3101         Returns:
3102             Tuple:
3103                 data from entry
3104                 data from fdtmap (excluding header)
3105                 Image object that was modified
3106         """
3107         dtb_data = self._DoReadFileDtb(dts, use_real_dtb=True,
3108                                        update_dtb=True)[1]
3109
3110         self.assertIn('image', control.images)
3111         image = control.images['image']
3112         entries = image.GetEntries()
3113         orig_dtb_data = entries['u-boot-dtb'].data
3114         orig_fdtmap_data = entries['fdtmap'].data
3115
3116         image_fname = tools.get_output_filename('image.bin')
3117         updated_fname = tools.get_output_filename('image-updated.bin')
3118         tools.write_file(updated_fname, tools.read_file(image_fname))
3119         image = control.WriteEntry(updated_fname, entry_name, data, decomp,
3120                                    allow_resize)
3121         data = control.ReadEntry(updated_fname, entry_name, decomp)
3122
3123         # The DT data should not change unless resized:
3124         if not allow_resize:
3125             new_dtb_data = entries['u-boot-dtb'].data
3126             self.assertEqual(new_dtb_data, orig_dtb_data)
3127             new_fdtmap_data = entries['fdtmap'].data
3128             self.assertEqual(new_fdtmap_data, orig_fdtmap_data)
3129
3130         return data, orig_fdtmap_data[fdtmap.FDTMAP_HDR_LEN:], image
3131
3132     def testReplaceSimple(self):
3133         """Test replacing a single file"""
3134         expected = b'x' * len(U_BOOT_DATA)
3135         data, expected_fdtmap, _ = self._RunReplaceCmd('u-boot', expected,
3136                                                     allow_resize=False)
3137         self.assertEqual(expected, data)
3138
3139         # Test that the state looks right. There should be an FDT for the fdtmap
3140         # that we jsut read back in, and it should match what we find in the
3141         # 'control' tables. Checking for an FDT that does not exist should
3142         # return None.
3143         path, fdtmap = state.GetFdtContents('fdtmap')
3144         self.assertIsNotNone(path)
3145         self.assertEqual(expected_fdtmap, fdtmap)
3146
3147         dtb = state.GetFdtForEtype('fdtmap')
3148         self.assertEqual(dtb.GetContents(), fdtmap)
3149
3150         missing_path, missing_fdtmap = state.GetFdtContents('missing')
3151         self.assertIsNone(missing_path)
3152         self.assertIsNone(missing_fdtmap)
3153
3154         missing_dtb = state.GetFdtForEtype('missing')
3155         self.assertIsNone(missing_dtb)
3156
3157         self.assertEqual('/binman', state.fdt_path_prefix)
3158
3159     def testReplaceResizeFail(self):
3160         """Test replacing a file by something larger"""
3161         expected = U_BOOT_DATA + b'x'
3162         with self.assertRaises(ValueError) as e:
3163             self._RunReplaceCmd('u-boot', expected, allow_resize=False,
3164                                 dts='139_replace_repack.dts')
3165         self.assertIn("Node '/u-boot': Entry data size does not match, but resize is disabled",
3166                       str(e.exception))
3167
3168     def testReplaceMulti(self):
3169         """Test replacing entry data where multiple images are generated"""
3170         data = self._DoReadFileDtb('133_replace_multi.dts', use_real_dtb=True,
3171                                    update_dtb=True)[0]
3172         expected = b'x' * len(U_BOOT_DATA)
3173         updated_fname = tools.get_output_filename('image-updated.bin')
3174         tools.write_file(updated_fname, data)
3175         entry_name = 'u-boot'
3176         control.WriteEntry(updated_fname, entry_name, expected,
3177                            allow_resize=False)
3178         data = control.ReadEntry(updated_fname, entry_name)
3179         self.assertEqual(expected, data)
3180
3181         # Check the state looks right.
3182         self.assertEqual('/binman/image', state.fdt_path_prefix)
3183
3184         # Now check we can write the first image
3185         image_fname = tools.get_output_filename('first-image.bin')
3186         updated_fname = tools.get_output_filename('first-updated.bin')
3187         tools.write_file(updated_fname, tools.read_file(image_fname))
3188         entry_name = 'u-boot'
3189         control.WriteEntry(updated_fname, entry_name, expected,
3190                            allow_resize=False)
3191         data = control.ReadEntry(updated_fname, entry_name)
3192         self.assertEqual(expected, data)
3193
3194         # Check the state looks right.
3195         self.assertEqual('/binman/first-image', state.fdt_path_prefix)
3196
3197     def testUpdateFdtAllRepack(self):
3198         """Test that all device trees are updated with offset/size info"""
3199         data = self._DoReadFileRealDtb('134_fdt_update_all_repack.dts')
3200         SECTION_SIZE = 0x300
3201         DTB_SIZE = 602
3202         FDTMAP_SIZE = 608
3203         base_expected = {
3204             'offset': 0,
3205             'size': SECTION_SIZE + DTB_SIZE * 2 + FDTMAP_SIZE,
3206             'image-pos': 0,
3207             'section:offset': 0,
3208             'section:size': SECTION_SIZE,
3209             'section:image-pos': 0,
3210             'section/u-boot-dtb:offset': 4,
3211             'section/u-boot-dtb:size': 636,
3212             'section/u-boot-dtb:image-pos': 4,
3213             'u-boot-spl-dtb:offset': SECTION_SIZE,
3214             'u-boot-spl-dtb:size': DTB_SIZE,
3215             'u-boot-spl-dtb:image-pos': SECTION_SIZE,
3216             'u-boot-tpl-dtb:offset': SECTION_SIZE + DTB_SIZE,
3217             'u-boot-tpl-dtb:image-pos': SECTION_SIZE + DTB_SIZE,
3218             'u-boot-tpl-dtb:size': DTB_SIZE,
3219             'fdtmap:offset': SECTION_SIZE + DTB_SIZE * 2,
3220             'fdtmap:size': FDTMAP_SIZE,
3221             'fdtmap:image-pos': SECTION_SIZE + DTB_SIZE * 2,
3222         }
3223         main_expected = {
3224             'section:orig-size': SECTION_SIZE,
3225             'section/u-boot-dtb:orig-offset': 4,
3226         }
3227
3228         # We expect three device-tree files in the output, with the first one
3229         # within a fixed-size section.
3230         # Read them in sequence. We look for an 'spl' property in the SPL tree,
3231         # and 'tpl' in the TPL tree, to make sure they are distinct from the
3232         # main U-Boot tree. All three should have the same positions and offset
3233         # except that the main tree should include the main_expected properties
3234         start = 4
3235         for item in ['', 'spl', 'tpl', None]:
3236             if item is None:
3237                 start += 16  # Move past fdtmap header
3238             dtb = fdt.Fdt.FromData(data[start:])
3239             dtb.Scan()
3240             props = self._GetPropTree(dtb,
3241                 BASE_DTB_PROPS + REPACK_DTB_PROPS + ['spl', 'tpl'],
3242                 prefix='/' if item is None else '/binman/')
3243             expected = dict(base_expected)
3244             if item:
3245                 expected[item] = 0
3246             else:
3247                 # Main DTB and fdtdec should include the 'orig-' properties
3248                 expected.update(main_expected)
3249             # Helpful for debugging:
3250             #for prop in sorted(props):
3251                 #print('prop %s %s %s' % (prop, props[prop], expected[prop]))
3252             self.assertEqual(expected, props)
3253             if item == '':
3254                 start = SECTION_SIZE
3255             else:
3256                 start += dtb._fdt_obj.totalsize()
3257
3258     def testFdtmapHeaderMiddle(self):
3259         """Test an FDT map in the middle of an image when it should be at end"""
3260         with self.assertRaises(ValueError) as e:
3261             self._DoReadFileRealDtb('135_fdtmap_hdr_middle.dts')
3262         self.assertIn("Invalid sibling order 'middle' for image-header: Must be at 'end' to match location",
3263                       str(e.exception))
3264
3265     def testFdtmapHeaderStartBad(self):
3266         """Test an FDT map in middle of an image when it should be at start"""
3267         with self.assertRaises(ValueError) as e:
3268             self._DoReadFileRealDtb('136_fdtmap_hdr_startbad.dts')
3269         self.assertIn("Invalid sibling order 'end' for image-header: Must be at 'start' to match location",
3270                       str(e.exception))
3271
3272     def testFdtmapHeaderEndBad(self):
3273         """Test an FDT map at the start of an image when it should be at end"""
3274         with self.assertRaises(ValueError) as e:
3275             self._DoReadFileRealDtb('137_fdtmap_hdr_endbad.dts')
3276         self.assertIn("Invalid sibling order 'start' for image-header: Must be at 'end' to match location",
3277                       str(e.exception))
3278
3279     def testFdtmapHeaderNoSize(self):
3280         """Test an image header at the end of an image with undefined size"""
3281         self._DoReadFileRealDtb('138_fdtmap_hdr_nosize.dts')
3282
3283     def testReplaceResize(self):
3284         """Test replacing a single file in an entry with a larger file"""
3285         expected = U_BOOT_DATA + b'x'
3286         data, _, image = self._RunReplaceCmd('u-boot', expected,
3287                                              dts='139_replace_repack.dts')
3288         self.assertEqual(expected, data)
3289
3290         entries = image.GetEntries()
3291         dtb_data = entries['u-boot-dtb'].data
3292         dtb = fdt.Fdt.FromData(dtb_data)
3293         dtb.Scan()
3294
3295         # The u-boot section should now be larger in the dtb
3296         node = dtb.GetNode('/binman/u-boot')
3297         self.assertEqual(len(expected), fdt_util.GetInt(node, 'size'))
3298
3299         # Same for the fdtmap
3300         fdata = entries['fdtmap'].data
3301         fdtb = fdt.Fdt.FromData(fdata[fdtmap.FDTMAP_HDR_LEN:])
3302         fdtb.Scan()
3303         fnode = fdtb.GetNode('/u-boot')
3304         self.assertEqual(len(expected), fdt_util.GetInt(fnode, 'size'))
3305
3306     def testReplaceResizeNoRepack(self):
3307         """Test replacing an entry with a larger file when not allowed"""
3308         expected = U_BOOT_DATA + b'x'
3309         with self.assertRaises(ValueError) as e:
3310             self._RunReplaceCmd('u-boot', expected)
3311         self.assertIn('Entry data size does not match, but allow-repack is not present for this image',
3312                       str(e.exception))
3313
3314     def testEntryShrink(self):
3315         """Test contracting an entry after it is packed"""
3316         try:
3317             state.SetAllowEntryContraction(True)
3318             data = self._DoReadFileDtb('140_entry_shrink.dts',
3319                                        update_dtb=True)[0]
3320         finally:
3321             state.SetAllowEntryContraction(False)
3322         self.assertEqual(b'a', data[:1])
3323         self.assertEqual(U_BOOT_DATA, data[1:1 + len(U_BOOT_DATA)])
3324         self.assertEqual(b'a', data[-1:])
3325
3326     def testEntryShrinkFail(self):
3327         """Test not being allowed to contract an entry after it is packed"""
3328         data = self._DoReadFileDtb('140_entry_shrink.dts', update_dtb=True)[0]
3329
3330         # In this case there is a spare byte at the end of the data. The size of
3331         # the contents is only 1 byte but we still have the size before it
3332         # shrunk.
3333         self.assertEqual(b'a\0', data[:2])
3334         self.assertEqual(U_BOOT_DATA, data[2:2 + len(U_BOOT_DATA)])
3335         self.assertEqual(b'a\0', data[-2:])
3336
3337     def testDescriptorOffset(self):
3338         """Test that the Intel descriptor is always placed at at the start"""
3339         data = self._DoReadFileDtb('141_descriptor_offset.dts')
3340         image = control.images['image']
3341         entries = image.GetEntries()
3342         desc = entries['intel-descriptor']
3343         self.assertEqual(0xff800000, desc.offset);
3344         self.assertEqual(0xff800000, desc.image_pos);
3345
3346     def testReplaceCbfs(self):
3347         """Test replacing a single file in CBFS without changing the size"""
3348         self._CheckLz4()
3349         expected = b'x' * len(U_BOOT_DATA)
3350         data = self._DoReadFileRealDtb('142_replace_cbfs.dts')
3351         updated_fname = tools.get_output_filename('image-updated.bin')
3352         tools.write_file(updated_fname, data)
3353         entry_name = 'section/cbfs/u-boot'
3354         control.WriteEntry(updated_fname, entry_name, expected,
3355                            allow_resize=True)
3356         data = control.ReadEntry(updated_fname, entry_name)
3357         self.assertEqual(expected, data)
3358
3359     def testReplaceResizeCbfs(self):
3360         """Test replacing a single file in CBFS with one of a different size"""
3361         self._CheckLz4()
3362         expected = U_BOOT_DATA + b'x'
3363         data = self._DoReadFileRealDtb('142_replace_cbfs.dts')
3364         updated_fname = tools.get_output_filename('image-updated.bin')
3365         tools.write_file(updated_fname, data)
3366         entry_name = 'section/cbfs/u-boot'
3367         control.WriteEntry(updated_fname, entry_name, expected,
3368                            allow_resize=True)
3369         data = control.ReadEntry(updated_fname, entry_name)
3370         self.assertEqual(expected, data)
3371
3372     def _SetupForReplace(self):
3373         """Set up some files to use to replace entries
3374
3375         This generates an image, copies it to a new file, extracts all the files
3376         in it and updates some of them
3377
3378         Returns:
3379             List
3380                 Image filename
3381                 Output directory
3382                 Expected values for updated entries, each a string
3383         """
3384         data = self._DoReadFileRealDtb('143_replace_all.dts')
3385
3386         updated_fname = tools.get_output_filename('image-updated.bin')
3387         tools.write_file(updated_fname, data)
3388
3389         outdir = os.path.join(self._indir, 'extract')
3390         einfos = control.ExtractEntries(updated_fname, None, outdir, [])
3391
3392         expected1 = b'x' + U_BOOT_DATA + b'y'
3393         u_boot_fname1 = os.path.join(outdir, 'u-boot')
3394         tools.write_file(u_boot_fname1, expected1)
3395
3396         expected2 = b'a' + U_BOOT_DATA + b'b'
3397         u_boot_fname2 = os.path.join(outdir, 'u-boot2')
3398         tools.write_file(u_boot_fname2, expected2)
3399
3400         expected_text = b'not the same text'
3401         text_fname = os.path.join(outdir, 'text')
3402         tools.write_file(text_fname, expected_text)
3403
3404         dtb_fname = os.path.join(outdir, 'u-boot-dtb')
3405         dtb = fdt.FdtScan(dtb_fname)
3406         node = dtb.GetNode('/binman/text')
3407         node.AddString('my-property', 'the value')
3408         dtb.Sync(auto_resize=True)
3409         dtb.Flush()
3410
3411         return updated_fname, outdir, expected1, expected2, expected_text
3412
3413     def _CheckReplaceMultiple(self, entry_paths):
3414         """Handle replacing the contents of multiple entries
3415
3416         Args:
3417             entry_paths: List of entry paths to replace
3418
3419         Returns:
3420             List
3421                 Dict of entries in the image:
3422                     key: Entry name
3423                     Value: Entry object
3424             Expected values for updated entries, each a string
3425         """
3426         updated_fname, outdir, expected1, expected2, expected_text = (
3427             self._SetupForReplace())
3428         control.ReplaceEntries(updated_fname, None, outdir, entry_paths)
3429
3430         image = Image.FromFile(updated_fname)
3431         image.LoadData()
3432         return image.GetEntries(), expected1, expected2, expected_text
3433
3434     def testReplaceAll(self):
3435         """Test replacing the contents of all entries"""
3436         entries, expected1, expected2, expected_text = (
3437             self._CheckReplaceMultiple([]))
3438         data = entries['u-boot'].data
3439         self.assertEqual(expected1, data)
3440
3441         data = entries['u-boot2'].data
3442         self.assertEqual(expected2, data)
3443
3444         data = entries['text'].data
3445         self.assertEqual(expected_text, data)
3446
3447         # Check that the device tree is updated
3448         data = entries['u-boot-dtb'].data
3449         dtb = fdt.Fdt.FromData(data)
3450         dtb.Scan()
3451         node = dtb.GetNode('/binman/text')
3452         self.assertEqual('the value', node.props['my-property'].value)
3453
3454     def testReplaceSome(self):
3455         """Test replacing the contents of a few entries"""
3456         entries, expected1, expected2, expected_text = (
3457             self._CheckReplaceMultiple(['u-boot2', 'text']))
3458
3459         # This one should not change
3460         data = entries['u-boot'].data
3461         self.assertEqual(U_BOOT_DATA, data)
3462
3463         data = entries['u-boot2'].data
3464         self.assertEqual(expected2, data)
3465
3466         data = entries['text'].data
3467         self.assertEqual(expected_text, data)
3468
3469     def testReplaceCmd(self):
3470         """Test replacing a file fron an image on the command line"""
3471         self._DoReadFileRealDtb('143_replace_all.dts')
3472
3473         try:
3474             tmpdir, updated_fname = self._SetupImageInTmpdir()
3475
3476             fname = os.path.join(tmpdir, 'update-u-boot.bin')
3477             expected = b'x' * len(U_BOOT_DATA)
3478             tools.write_file(fname, expected)
3479
3480             self._DoBinman('replace', '-i', updated_fname, 'u-boot', '-f', fname)
3481             data = tools.read_file(updated_fname)
3482             self.assertEqual(expected, data[:len(expected)])
3483             map_fname = os.path.join(tmpdir, 'image-updated.map')
3484             self.assertFalse(os.path.exists(map_fname))
3485         finally:
3486             shutil.rmtree(tmpdir)
3487
3488     def testReplaceCmdSome(self):
3489         """Test replacing some files fron an image on the command line"""
3490         updated_fname, outdir, expected1, expected2, expected_text = (
3491             self._SetupForReplace())
3492
3493         self._DoBinman('replace', '-i', updated_fname, '-I', outdir,
3494                        'u-boot2', 'text')
3495
3496         tools.prepare_output_dir(None)
3497         image = Image.FromFile(updated_fname)
3498         image.LoadData()
3499         entries = image.GetEntries()
3500
3501         # This one should not change
3502         data = entries['u-boot'].data
3503         self.assertEqual(U_BOOT_DATA, data)
3504
3505         data = entries['u-boot2'].data
3506         self.assertEqual(expected2, data)
3507
3508         data = entries['text'].data
3509         self.assertEqual(expected_text, data)
3510
3511     def testReplaceMissing(self):
3512         """Test replacing entries where the file is missing"""
3513         updated_fname, outdir, expected1, expected2, expected_text = (
3514             self._SetupForReplace())
3515
3516         # Remove one of the files, to generate a warning
3517         u_boot_fname1 = os.path.join(outdir, 'u-boot')
3518         os.remove(u_boot_fname1)
3519
3520         with test_util.capture_sys_output() as (stdout, stderr):
3521             control.ReplaceEntries(updated_fname, None, outdir, [])
3522         self.assertIn("Skipping entry '/u-boot' from missing file",
3523                       stderr.getvalue())
3524
3525     def testReplaceCmdMap(self):
3526         """Test replacing a file fron an image on the command line"""
3527         self._DoReadFileRealDtb('143_replace_all.dts')
3528
3529         try:
3530             tmpdir, updated_fname = self._SetupImageInTmpdir()
3531
3532             fname = os.path.join(self._indir, 'update-u-boot.bin')
3533             expected = b'x' * len(U_BOOT_DATA)
3534             tools.write_file(fname, expected)
3535
3536             self._DoBinman('replace', '-i', updated_fname, 'u-boot',
3537                            '-f', fname, '-m')
3538             map_fname = os.path.join(tmpdir, 'image-updated.map')
3539             self.assertTrue(os.path.exists(map_fname))
3540         finally:
3541             shutil.rmtree(tmpdir)
3542
3543     def testReplaceNoEntryPaths(self):
3544         """Test replacing an entry without an entry path"""
3545         self._DoReadFileRealDtb('143_replace_all.dts')
3546         image_fname = tools.get_output_filename('image.bin')
3547         with self.assertRaises(ValueError) as e:
3548             control.ReplaceEntries(image_fname, 'fname', None, [])
3549         self.assertIn('Must specify an entry path to read with -f',
3550                       str(e.exception))
3551
3552     def testReplaceTooManyEntryPaths(self):
3553         """Test extracting some entries"""
3554         self._DoReadFileRealDtb('143_replace_all.dts')
3555         image_fname = tools.get_output_filename('image.bin')
3556         with self.assertRaises(ValueError) as e:
3557             control.ReplaceEntries(image_fname, 'fname', None, ['a', 'b'])
3558         self.assertIn('Must specify exactly one entry path to write with -f',
3559                       str(e.exception))
3560
3561     def testPackReset16(self):
3562         """Test that an image with an x86 reset16 region can be created"""
3563         data = self._DoReadFile('144_x86_reset16.dts')
3564         self.assertEqual(X86_RESET16_DATA, data[:len(X86_RESET16_DATA)])
3565
3566     def testPackReset16Spl(self):
3567         """Test that an image with an x86 reset16-spl region can be created"""
3568         data = self._DoReadFile('145_x86_reset16_spl.dts')
3569         self.assertEqual(X86_RESET16_SPL_DATA, data[:len(X86_RESET16_SPL_DATA)])
3570
3571     def testPackReset16Tpl(self):
3572         """Test that an image with an x86 reset16-tpl region can be created"""
3573         data = self._DoReadFile('146_x86_reset16_tpl.dts')
3574         self.assertEqual(X86_RESET16_TPL_DATA, data[:len(X86_RESET16_TPL_DATA)])
3575
3576     def testPackIntelFit(self):
3577         """Test that an image with an Intel FIT and pointer can be created"""
3578         data = self._DoReadFile('147_intel_fit.dts')
3579         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
3580         fit = data[16:32];
3581         self.assertEqual(b'_FIT_   \x01\x00\x00\x00\x00\x01\x80}' , fit)
3582         ptr = struct.unpack('<i', data[0x40:0x44])[0]
3583
3584         image = control.images['image']
3585         entries = image.GetEntries()
3586         expected_ptr = entries['intel-fit'].image_pos - (1 << 32)
3587         self.assertEqual(expected_ptr, ptr)
3588
3589     def testPackIntelFitMissing(self):
3590         """Test detection of a FIT pointer with not FIT region"""
3591         with self.assertRaises(ValueError) as e:
3592             self._DoReadFile('148_intel_fit_missing.dts')
3593         self.assertIn("'intel-fit-ptr' section must have an 'intel-fit' sibling",
3594                       str(e.exception))
3595
3596     def _CheckSymbolsTplSection(self, dts, expected_vals):
3597         data = self._DoReadFile(dts)
3598         sym_values = struct.pack('<LQLL', *expected_vals)
3599         upto1 = 4 + len(U_BOOT_SPL_DATA)
3600         expected1 = tools.get_bytes(0xff, 4) + sym_values + U_BOOT_SPL_DATA[20:]
3601         self.assertEqual(expected1, data[:upto1])
3602
3603         upto2 = upto1 + 1 + len(U_BOOT_SPL_DATA)
3604         expected2 = tools.get_bytes(0xff, 1) + sym_values + U_BOOT_SPL_DATA[20:]
3605         self.assertEqual(expected2, data[upto1:upto2])
3606
3607         upto3 = 0x34 + len(U_BOOT_DATA)
3608         expected3 = tools.get_bytes(0xff, 1) + U_BOOT_DATA
3609         self.assertEqual(expected3, data[upto2:upto3])
3610
3611         expected4 = sym_values + U_BOOT_TPL_DATA[20:]
3612         self.assertEqual(expected4, data[upto3:upto3 + len(U_BOOT_TPL_DATA)])
3613
3614     def testSymbolsTplSection(self):
3615         """Test binman can assign symbols embedded in U-Boot TPL in a section"""
3616         self._SetupSplElf('u_boot_binman_syms')
3617         self._SetupTplElf('u_boot_binman_syms')
3618         self._CheckSymbolsTplSection('149_symbols_tpl.dts',
3619                                      [0x04, 0x1c, 0x10 + 0x34, 0x04])
3620
3621     def testSymbolsTplSectionX86(self):
3622         """Test binman can assign symbols in a section with end-at-4gb"""
3623         self._SetupSplElf('u_boot_binman_syms_x86')
3624         self._SetupTplElf('u_boot_binman_syms_x86')
3625         self._CheckSymbolsTplSection('155_symbols_tpl_x86.dts',
3626                                      [0xffffff04, 0xffffff1c, 0xffffff34,
3627                                       0x04])
3628
3629     def testPackX86RomIfwiSectiom(self):
3630         """Test that a section can be placed in an IFWI region"""
3631         self._SetupIfwi('fitimage.bin')
3632         data = self._DoReadFile('151_x86_rom_ifwi_section.dts')
3633         self._CheckIfwi(data)
3634
3635     def testPackFspM(self):
3636         """Test that an image with a FSP memory-init binary can be created"""
3637         data = self._DoReadFile('152_intel_fsp_m.dts')
3638         self.assertEqual(FSP_M_DATA, data[:len(FSP_M_DATA)])
3639
3640     def testPackFspS(self):
3641         """Test that an image with a FSP silicon-init binary can be created"""
3642         data = self._DoReadFile('153_intel_fsp_s.dts')
3643         self.assertEqual(FSP_S_DATA, data[:len(FSP_S_DATA)])
3644
3645     def testPackFspT(self):
3646         """Test that an image with a FSP temp-ram-init binary can be created"""
3647         data = self._DoReadFile('154_intel_fsp_t.dts')
3648         self.assertEqual(FSP_T_DATA, data[:len(FSP_T_DATA)])
3649
3650     def testMkimage(self):
3651         """Test using mkimage to build an image"""
3652         data = self._DoReadFile('156_mkimage.dts')
3653
3654         # Just check that the data appears in the file somewhere
3655         self.assertIn(U_BOOT_SPL_DATA, data)
3656
3657     def testMkimageMissing(self):
3658         """Test that binman still produces an image if mkimage is missing"""
3659         with test_util.capture_sys_output() as (_, stderr):
3660             self._DoTestFile('156_mkimage.dts',
3661                              force_missing_bintools='mkimage')
3662         err = stderr.getvalue()
3663         self.assertRegex(err,
3664                          "Image 'main-section'.*missing bintools.*: mkimage")
3665
3666     def testExtblob(self):
3667         """Test an image with an external blob"""
3668         data = self._DoReadFile('157_blob_ext.dts')
3669         self.assertEqual(REFCODE_DATA, data)
3670
3671     def testExtblobMissing(self):
3672         """Test an image with a missing external blob"""
3673         with self.assertRaises(ValueError) as e:
3674             self._DoReadFile('158_blob_ext_missing.dts')
3675         self.assertIn("Filename 'missing-file' not found in input path",
3676                       str(e.exception))
3677
3678     def testExtblobMissingOk(self):
3679         """Test an image with an missing external blob that is allowed"""
3680         with test_util.capture_sys_output() as (stdout, stderr):
3681             self._DoTestFile('158_blob_ext_missing.dts', allow_missing=True)
3682         err = stderr.getvalue()
3683         self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext")
3684
3685     def testExtblobMissingOkSect(self):
3686         """Test an image with an missing external blob that is allowed"""
3687         with test_util.capture_sys_output() as (stdout, stderr):
3688             self._DoTestFile('159_blob_ext_missing_sect.dts',
3689                              allow_missing=True)
3690         err = stderr.getvalue()
3691         self.assertRegex(err, "Image 'main-section'.*missing.*: "
3692                          "blob-ext blob-ext2")
3693
3694     def testPackX86RomMeMissingDesc(self):
3695         """Test that an missing Intel descriptor entry is allowed"""
3696         with test_util.capture_sys_output() as (stdout, stderr):
3697             self._DoTestFile('164_x86_rom_me_missing.dts', allow_missing=True)
3698         err = stderr.getvalue()
3699         self.assertRegex(err,
3700                          "Image 'main-section'.*missing.*: intel-descriptor")
3701
3702     def testPackX86RomMissingIfwi(self):
3703         """Test that an x86 ROM with Integrated Firmware Image can be created"""
3704         self._SetupIfwi('fitimage.bin')
3705         pathname = os.path.join(self._indir, 'fitimage.bin')
3706         os.remove(pathname)
3707         with test_util.capture_sys_output() as (stdout, stderr):
3708             self._DoTestFile('111_x86_rom_ifwi.dts', allow_missing=True)
3709         err = stderr.getvalue()
3710         self.assertRegex(err, "Image 'main-section'.*missing.*: intel-ifwi")
3711
3712     def testPackOverlap(self):
3713         """Test that zero-size overlapping regions are ignored"""
3714         self._DoTestFile('160_pack_overlap_zero.dts')
3715
3716     def testSimpleFit(self):
3717         """Test an image with a FIT inside"""
3718         data = self._DoReadFile('161_fit.dts')
3719         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
3720         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
3721         fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
3722
3723         # The data should be inside the FIT
3724         dtb = fdt.Fdt.FromData(fit_data)
3725         dtb.Scan()
3726         fnode = dtb.GetNode('/images/kernel')
3727         self.assertIn('data', fnode.props)
3728
3729         fname = os.path.join(self._indir, 'fit_data.fit')
3730         tools.write_file(fname, fit_data)
3731         out = tools.run('dumpimage', '-l', fname)
3732
3733         # Check a few features to make sure the plumbing works. We don't need
3734         # to test the operation of mkimage or dumpimage here. First convert the
3735         # output into a dict where the keys are the fields printed by dumpimage
3736         # and the values are a list of values for each field
3737         lines = out.splitlines()
3738
3739         # Converts "Compression:  gzip compressed" into two groups:
3740         # 'Compression' and 'gzip compressed'
3741         re_line = re.compile(r'^ *([^:]*)(?:: *(.*))?$')
3742         vals = collections.defaultdict(list)
3743         for line in lines:
3744             mat = re_line.match(line)
3745             vals[mat.group(1)].append(mat.group(2))
3746
3747         self.assertEquals('FIT description: test-desc', lines[0])
3748         self.assertIn('Created:', lines[1])
3749         self.assertIn('Image 0 (kernel)', vals)
3750         self.assertIn('Hash value', vals)
3751         data_sizes = vals.get('Data Size')
3752         self.assertIsNotNone(data_sizes)
3753         self.assertEqual(2, len(data_sizes))
3754         # Format is "4 Bytes = 0.00 KiB = 0.00 MiB" so take the first word
3755         self.assertEqual(len(U_BOOT_DATA), int(data_sizes[0].split()[0]))
3756         self.assertEqual(len(U_BOOT_SPL_DTB_DATA), int(data_sizes[1].split()[0]))
3757
3758     def testFitExternal(self):
3759         """Test an image with an FIT with external images"""
3760         data = self._DoReadFile('162_fit_external.dts')
3761         fit_data = data[len(U_BOOT_DATA):-2]  # _testing is 2 bytes
3762
3763         # Size of the external-data region as set up by mkimage
3764         external_data_size = len(U_BOOT_DATA) + 2
3765         expected_size = (len(U_BOOT_DATA) + 0x400 +
3766                          tools.align(external_data_size, 4) +
3767                          len(U_BOOT_NODTB_DATA))
3768
3769         # The data should be outside the FIT
3770         dtb = fdt.Fdt.FromData(fit_data)
3771         dtb.Scan()
3772         fnode = dtb.GetNode('/images/kernel')
3773         self.assertNotIn('data', fnode.props)
3774         self.assertEqual(len(U_BOOT_DATA),
3775                          fdt_util.fdt32_to_cpu(fnode.props['data-size'].value))
3776         fit_pos = 0x400;
3777         self.assertEqual(
3778             fit_pos,
3779             fdt_util.fdt32_to_cpu(fnode.props['data-position'].value))
3780
3781         self.assertEquals(expected_size, len(data))
3782         actual_pos = len(U_BOOT_DATA) + fit_pos
3783         self.assertEqual(U_BOOT_DATA + b'aa',
3784                          data[actual_pos:actual_pos + external_data_size])
3785
3786     def testFitMissing(self):
3787         """Test that binman still produces a FIT image if mkimage is missing"""
3788         with test_util.capture_sys_output() as (_, stderr):
3789             self._DoTestFile('162_fit_external.dts',
3790                              force_missing_bintools='mkimage')
3791         err = stderr.getvalue()
3792         self.assertRegex(err,
3793                          "Image 'main-section'.*missing bintools.*: mkimage")
3794
3795     def testSectionIgnoreHashSignature(self):
3796         """Test that sections ignore hash, signature nodes for its data"""
3797         data = self._DoReadFile('165_section_ignore_hash_signature.dts')
3798         expected = (U_BOOT_DATA + U_BOOT_DATA)
3799         self.assertEqual(expected, data)
3800
3801     def testPadInSections(self):
3802         """Test pad-before, pad-after for entries in sections"""
3803         data, _, _, out_dtb_fname = self._DoReadFileDtb(
3804             '166_pad_in_sections.dts', update_dtb=True)
3805         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
3806                     U_BOOT_DATA + tools.get_bytes(ord('!'), 6) +
3807                     U_BOOT_DATA)
3808         self.assertEqual(expected, data)
3809
3810         dtb = fdt.Fdt(out_dtb_fname)
3811         dtb.Scan()
3812         props = self._GetPropTree(dtb, ['size', 'image-pos', 'offset'])
3813         expected = {
3814             'image-pos': 0,
3815             'offset': 0,
3816             'size': 12 + 6 + 3 * len(U_BOOT_DATA),
3817
3818             'section:image-pos': 0,
3819             'section:offset': 0,
3820             'section:size': 12 + 6 + 3 * len(U_BOOT_DATA),
3821
3822             'section/before:image-pos': 0,
3823             'section/before:offset': 0,
3824             'section/before:size': len(U_BOOT_DATA),
3825
3826             'section/u-boot:image-pos': 4,
3827             'section/u-boot:offset': 4,
3828             'section/u-boot:size': 12 + len(U_BOOT_DATA) + 6,
3829
3830             'section/after:image-pos': 26,
3831             'section/after:offset': 26,
3832             'section/after:size': len(U_BOOT_DATA),
3833             }
3834         self.assertEqual(expected, props)
3835
3836     def testFitImageSubentryAlignment(self):
3837         """Test relative alignability of FIT image subentries"""
3838         entry_args = {
3839             'test-id': TEXT_DATA,
3840         }
3841         data, _, _, _ = self._DoReadFileDtb('167_fit_image_subentry_alignment.dts',
3842                                             entry_args=entry_args)
3843         dtb = fdt.Fdt.FromData(data)
3844         dtb.Scan()
3845
3846         node = dtb.GetNode('/images/kernel')
3847         data = dtb.GetProps(node)["data"].bytes
3848         align_pad = 0x10 - (len(U_BOOT_SPL_DATA) % 0x10)
3849         expected = (tools.get_bytes(0, 0x20) + U_BOOT_SPL_DATA +
3850                     tools.get_bytes(0, align_pad) + U_BOOT_DATA)
3851         self.assertEqual(expected, data)
3852
3853         node = dtb.GetNode('/images/fdt-1')
3854         data = dtb.GetProps(node)["data"].bytes
3855         expected = (U_BOOT_SPL_DTB_DATA + tools.get_bytes(0, 20) +
3856                     tools.to_bytes(TEXT_DATA) + tools.get_bytes(0, 30) +
3857                     U_BOOT_DTB_DATA)
3858         self.assertEqual(expected, data)
3859
3860     def testFitExtblobMissingOk(self):
3861         """Test a FIT with a missing external blob that is allowed"""
3862         with test_util.capture_sys_output() as (stdout, stderr):
3863             self._DoTestFile('168_fit_missing_blob.dts',
3864                              allow_missing=True)
3865         err = stderr.getvalue()
3866         self.assertRegex(err, "Image 'main-section'.*missing.*: atf-bl31")
3867
3868     def testBlobNamedByArgMissing(self):
3869         """Test handling of a missing entry arg"""
3870         with self.assertRaises(ValueError) as e:
3871             self._DoReadFile('068_blob_named_by_arg.dts')
3872         self.assertIn("Missing required properties/entry args: cros-ec-rw-path",
3873                       str(e.exception))
3874
3875     def testPackBl31(self):
3876         """Test that an image with an ATF BL31 binary can be created"""
3877         data = self._DoReadFile('169_atf_bl31.dts')
3878         self.assertEqual(ATF_BL31_DATA, data[:len(ATF_BL31_DATA)])
3879
3880     def testPackScp(self):
3881         """Test that an image with an SCP binary can be created"""
3882         data = self._DoReadFile('172_scp.dts')
3883         self.assertEqual(SCP_DATA, data[:len(SCP_DATA)])
3884
3885     def testFitFdt(self):
3886         """Test an image with an FIT with multiple FDT images"""
3887         def _CheckFdt(seq, expected_data):
3888             """Check the FDT nodes
3889
3890             Args:
3891                 seq: Sequence number to check (0 or 1)
3892                 expected_data: Expected contents of 'data' property
3893             """
3894             name = 'fdt-%d' % seq
3895             fnode = dtb.GetNode('/images/%s' % name)
3896             self.assertIsNotNone(fnode)
3897             self.assertEqual({'description','type', 'compression', 'data'},
3898                              set(fnode.props.keys()))
3899             self.assertEqual(expected_data, fnode.props['data'].bytes)
3900             self.assertEqual('fdt-test-fdt%d.dtb' % seq,
3901                              fnode.props['description'].value)
3902
3903         def _CheckConfig(seq, expected_data):
3904             """Check the configuration nodes
3905
3906             Args:
3907                 seq: Sequence number to check (0 or 1)
3908                 expected_data: Expected contents of 'data' property
3909             """
3910             cnode = dtb.GetNode('/configurations')
3911             self.assertIn('default', cnode.props)
3912             self.assertEqual('config-2', cnode.props['default'].value)
3913
3914             name = 'config-%d' % seq
3915             fnode = dtb.GetNode('/configurations/%s' % name)
3916             self.assertIsNotNone(fnode)
3917             self.assertEqual({'description','firmware', 'loadables', 'fdt'},
3918                              set(fnode.props.keys()))
3919             self.assertEqual('conf-test-fdt%d.dtb' % seq,
3920                              fnode.props['description'].value)
3921             self.assertEqual('fdt-%d' % seq, fnode.props['fdt'].value)
3922
3923         entry_args = {
3924             'of-list': 'test-fdt1 test-fdt2',
3925             'default-dt': 'test-fdt2',
3926         }
3927         data = self._DoReadFileDtb(
3928             '170_fit_fdt.dts',
3929             entry_args=entry_args,
3930             extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
3931         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
3932         fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
3933
3934         dtb = fdt.Fdt.FromData(fit_data)
3935         dtb.Scan()
3936         fnode = dtb.GetNode('/images/kernel')
3937         self.assertIn('data', fnode.props)
3938
3939         # Check all the properties in fdt-1 and fdt-2
3940         _CheckFdt(1, TEST_FDT1_DATA)
3941         _CheckFdt(2, TEST_FDT2_DATA)
3942
3943         # Check configurations
3944         _CheckConfig(1, TEST_FDT1_DATA)
3945         _CheckConfig(2, TEST_FDT2_DATA)
3946
3947     def testFitFdtMissingList(self):
3948         """Test handling of a missing 'of-list' entry arg"""
3949         with self.assertRaises(ValueError) as e:
3950             self._DoReadFile('170_fit_fdt.dts')
3951         self.assertIn("Generator node requires 'of-list' entry argument",
3952                       str(e.exception))
3953
3954     def testFitFdtEmptyList(self):
3955         """Test handling of an empty 'of-list' entry arg"""
3956         entry_args = {
3957             'of-list': '',
3958         }
3959         data = self._DoReadFileDtb('170_fit_fdt.dts', entry_args=entry_args)[0]
3960
3961     def testFitFdtMissingProp(self):
3962         """Test handling of a missing 'fit,fdt-list' property"""
3963         with self.assertRaises(ValueError) as e:
3964             self._DoReadFile('171_fit_fdt_missing_prop.dts')
3965         self.assertIn("Generator node requires 'fit,fdt-list' property",
3966                       str(e.exception))
3967
3968     def testFitFdtEmptyList(self):
3969         """Test handling of an empty 'of-list' entry arg"""
3970         entry_args = {
3971             'of-list': '',
3972         }
3973         data = self._DoReadFileDtb('170_fit_fdt.dts', entry_args=entry_args)[0]
3974
3975     def testFitFdtMissing(self):
3976         """Test handling of a missing 'default-dt' entry arg"""
3977         entry_args = {
3978             'of-list': 'test-fdt1 test-fdt2',
3979         }
3980         with self.assertRaises(ValueError) as e:
3981             self._DoReadFileDtb(
3982                 '170_fit_fdt.dts',
3983                 entry_args=entry_args,
3984                 extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
3985         self.assertIn("Generated 'default' node requires default-dt entry argument",
3986                       str(e.exception))
3987
3988     def testFitFdtNotInList(self):
3989         """Test handling of a default-dt that is not in the of-list"""
3990         entry_args = {
3991             'of-list': 'test-fdt1 test-fdt2',
3992             'default-dt': 'test-fdt3',
3993         }
3994         with self.assertRaises(ValueError) as e:
3995             self._DoReadFileDtb(
3996                 '170_fit_fdt.dts',
3997                 entry_args=entry_args,
3998                 extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0]
3999         self.assertIn("default-dt entry argument 'test-fdt3' not found in fdt list: test-fdt1, test-fdt2",
4000                       str(e.exception))
4001
4002     def testFitExtblobMissingHelp(self):
4003         """Test display of help messages when an external blob is missing"""
4004         control.missing_blob_help = control._ReadMissingBlobHelp()
4005         control.missing_blob_help['wibble'] = 'Wibble test'
4006         control.missing_blob_help['another'] = 'Another test'
4007         with test_util.capture_sys_output() as (stdout, stderr):
4008             self._DoTestFile('168_fit_missing_blob.dts',
4009                              allow_missing=True)
4010         err = stderr.getvalue()
4011
4012         # We can get the tag from the name, the type or the missing-msg
4013         # property. Check all three.
4014         self.assertIn('You may need to build ARM Trusted', err)
4015         self.assertIn('Wibble test', err)
4016         self.assertIn('Another test', err)
4017
4018     def testMissingBlob(self):
4019         """Test handling of a blob containing a missing file"""
4020         with self.assertRaises(ValueError) as e:
4021             self._DoTestFile('173_missing_blob.dts', allow_missing=True)
4022         self.assertIn("Filename 'missing' not found in input path",
4023                       str(e.exception))
4024
4025     def testEnvironment(self):
4026         """Test adding a U-Boot environment"""
4027         data = self._DoReadFile('174_env.dts')
4028         self.assertEqual(U_BOOT_DATA, data[:len(U_BOOT_DATA)])
4029         self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):])
4030         env = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)]
4031         self.assertEqual(b'\x1b\x97\x22\x7c\x01var1=1\0var2="2"\0\0\xff\xff',
4032                          env)
4033
4034     def testEnvironmentNoSize(self):
4035         """Test that a missing 'size' property is detected"""
4036         with self.assertRaises(ValueError) as e:
4037             self._DoTestFile('175_env_no_size.dts')
4038         self.assertIn("'u-boot-env' entry must have a size property",
4039                       str(e.exception))
4040
4041     def testEnvironmentTooSmall(self):
4042         """Test handling of an environment that does not fit"""
4043         with self.assertRaises(ValueError) as e:
4044             self._DoTestFile('176_env_too_small.dts')
4045
4046         # checksum, start byte, environment with \0 terminator, final \0
4047         need = 4 + 1 + len(ENV_DATA) + 1 + 1
4048         short = need - 0x8
4049         self.assertIn("too small to hold data (need %#x more bytes)" % short,
4050                       str(e.exception))
4051
4052     def testSkipAtStart(self):
4053         """Test handling of skip-at-start section"""
4054         data = self._DoReadFile('177_skip_at_start.dts')
4055         self.assertEqual(U_BOOT_DATA, data)
4056
4057         image = control.images['image']
4058         entries = image.GetEntries()
4059         section = entries['section']
4060         self.assertEqual(0, section.offset)
4061         self.assertEqual(len(U_BOOT_DATA), section.size)
4062         self.assertEqual(U_BOOT_DATA, section.GetData())
4063
4064         entry = section.GetEntries()['u-boot']
4065         self.assertEqual(16, entry.offset)
4066         self.assertEqual(len(U_BOOT_DATA), entry.size)
4067         self.assertEqual(U_BOOT_DATA, entry.data)
4068
4069     def testSkipAtStartPad(self):
4070         """Test handling of skip-at-start section with padded entry"""
4071         data = self._DoReadFile('178_skip_at_start_pad.dts')
4072         before = tools.get_bytes(0, 8)
4073         after = tools.get_bytes(0, 4)
4074         all = before + U_BOOT_DATA + after
4075         self.assertEqual(all, data)
4076
4077         image = control.images['image']
4078         entries = image.GetEntries()
4079         section = entries['section']
4080         self.assertEqual(0, section.offset)
4081         self.assertEqual(len(all), section.size)
4082         self.assertEqual(all, section.GetData())
4083
4084         entry = section.GetEntries()['u-boot']
4085         self.assertEqual(16, entry.offset)
4086         self.assertEqual(len(all), entry.size)
4087         self.assertEqual(U_BOOT_DATA, entry.data)
4088
4089     def testSkipAtStartSectionPad(self):
4090         """Test handling of skip-at-start section with padding"""
4091         data = self._DoReadFile('179_skip_at_start_section_pad.dts')
4092         before = tools.get_bytes(0, 8)
4093         after = tools.get_bytes(0, 4)
4094         all = before + U_BOOT_DATA + after
4095         self.assertEqual(all, data)
4096
4097         image = control.images['image']
4098         entries = image.GetEntries()
4099         section = entries['section']
4100         self.assertEqual(0, section.offset)
4101         self.assertEqual(len(all), section.size)
4102         self.assertEqual(U_BOOT_DATA, section.data)
4103         self.assertEqual(all, section.GetPaddedData())
4104
4105         entry = section.GetEntries()['u-boot']
4106         self.assertEqual(16, entry.offset)
4107         self.assertEqual(len(U_BOOT_DATA), entry.size)
4108         self.assertEqual(U_BOOT_DATA, entry.data)
4109
4110     def testSectionPad(self):
4111         """Testing padding with sections"""
4112         data = self._DoReadFile('180_section_pad.dts')
4113         expected = (tools.get_bytes(ord('&'), 3) +
4114                     tools.get_bytes(ord('!'), 5) +
4115                     U_BOOT_DATA +
4116                     tools.get_bytes(ord('!'), 1) +
4117                     tools.get_bytes(ord('&'), 2))
4118         self.assertEqual(expected, data)
4119
4120     def testSectionAlign(self):
4121         """Testing alignment with sections"""
4122         data = self._DoReadFileDtb('181_section_align.dts', map=True)[0]
4123         expected = (b'\0' +                         # fill section
4124                     tools.get_bytes(ord('&'), 1) +   # padding to section align
4125                     b'\0' +                         # fill section
4126                     tools.get_bytes(ord('!'), 3) +   # padding to u-boot align
4127                     U_BOOT_DATA +
4128                     tools.get_bytes(ord('!'), 4) +   # padding to u-boot size
4129                     tools.get_bytes(ord('!'), 4))    # padding to section size
4130         self.assertEqual(expected, data)
4131
4132     def testCompressImage(self):
4133         """Test compression of the entire image"""
4134         self._CheckLz4()
4135         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4136             '182_compress_image.dts', use_real_dtb=True, update_dtb=True)
4137         dtb = fdt.Fdt(out_dtb_fname)
4138         dtb.Scan()
4139         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4140                                         'uncomp-size'])
4141         orig = self._decompress(data)
4142         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4143
4144         # Do a sanity check on various fields
4145         image = control.images['image']
4146         entries = image.GetEntries()
4147         self.assertEqual(2, len(entries))
4148
4149         entry = entries['blob']
4150         self.assertEqual(COMPRESS_DATA, entry.data)
4151         self.assertEqual(len(COMPRESS_DATA), entry.size)
4152
4153         entry = entries['u-boot']
4154         self.assertEqual(U_BOOT_DATA, entry.data)
4155         self.assertEqual(len(U_BOOT_DATA), entry.size)
4156
4157         self.assertEqual(len(data), image.size)
4158         self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, image.uncomp_data)
4159         self.assertEqual(len(COMPRESS_DATA + U_BOOT_DATA), image.uncomp_size)
4160         orig = self._decompress(image.data)
4161         self.assertEqual(orig, image.uncomp_data)
4162
4163         expected = {
4164             'blob:offset': 0,
4165             'blob:size': len(COMPRESS_DATA),
4166             'u-boot:offset': len(COMPRESS_DATA),
4167             'u-boot:size': len(U_BOOT_DATA),
4168             'uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4169             'offset': 0,
4170             'image-pos': 0,
4171             'size': len(data),
4172             }
4173         self.assertEqual(expected, props)
4174
4175     def testCompressImageLess(self):
4176         """Test compression where compression reduces the image size"""
4177         self._CheckLz4()
4178         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4179             '183_compress_image_less.dts', use_real_dtb=True, update_dtb=True)
4180         dtb = fdt.Fdt(out_dtb_fname)
4181         dtb.Scan()
4182         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4183                                         'uncomp-size'])
4184         orig = self._decompress(data)
4185
4186         self.assertEquals(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig)
4187
4188         # Do a sanity check on various fields
4189         image = control.images['image']
4190         entries = image.GetEntries()
4191         self.assertEqual(2, len(entries))
4192
4193         entry = entries['blob']
4194         self.assertEqual(COMPRESS_DATA_BIG, entry.data)
4195         self.assertEqual(len(COMPRESS_DATA_BIG), entry.size)
4196
4197         entry = entries['u-boot']
4198         self.assertEqual(U_BOOT_DATA, entry.data)
4199         self.assertEqual(len(U_BOOT_DATA), entry.size)
4200
4201         self.assertEqual(len(data), image.size)
4202         self.assertEqual(COMPRESS_DATA_BIG + U_BOOT_DATA, image.uncomp_data)
4203         self.assertEqual(len(COMPRESS_DATA_BIG + U_BOOT_DATA),
4204                          image.uncomp_size)
4205         orig = self._decompress(image.data)
4206         self.assertEqual(orig, image.uncomp_data)
4207
4208         expected = {
4209             'blob:offset': 0,
4210             'blob:size': len(COMPRESS_DATA_BIG),
4211             'u-boot:offset': len(COMPRESS_DATA_BIG),
4212             'u-boot:size': len(U_BOOT_DATA),
4213             'uncomp-size': len(COMPRESS_DATA_BIG + U_BOOT_DATA),
4214             'offset': 0,
4215             'image-pos': 0,
4216             'size': len(data),
4217             }
4218         self.assertEqual(expected, props)
4219
4220     def testCompressSectionSize(self):
4221         """Test compression of a section with a fixed size"""
4222         self._CheckLz4()
4223         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4224             '184_compress_section_size.dts', use_real_dtb=True, update_dtb=True)
4225         dtb = fdt.Fdt(out_dtb_fname)
4226         dtb.Scan()
4227         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4228                                         'uncomp-size'])
4229         orig = self._decompress(data)
4230         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4231         expected = {
4232             'section/blob:offset': 0,
4233             'section/blob:size': len(COMPRESS_DATA),
4234             'section/u-boot:offset': len(COMPRESS_DATA),
4235             'section/u-boot:size': len(U_BOOT_DATA),
4236             'section:offset': 0,
4237             'section:image-pos': 0,
4238             'section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4239             'section:size': 0x30,
4240             'offset': 0,
4241             'image-pos': 0,
4242             'size': 0x30,
4243             }
4244         self.assertEqual(expected, props)
4245
4246     def testCompressSection(self):
4247         """Test compression of a section with no fixed size"""
4248         self._CheckLz4()
4249         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4250             '185_compress_section.dts', use_real_dtb=True, update_dtb=True)
4251         dtb = fdt.Fdt(out_dtb_fname)
4252         dtb.Scan()
4253         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4254                                         'uncomp-size'])
4255         orig = self._decompress(data)
4256         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig)
4257         expected = {
4258             'section/blob:offset': 0,
4259             'section/blob:size': len(COMPRESS_DATA),
4260             'section/u-boot:offset': len(COMPRESS_DATA),
4261             'section/u-boot:size': len(U_BOOT_DATA),
4262             'section:offset': 0,
4263             'section:image-pos': 0,
4264             'section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4265             'section:size': len(data),
4266             'offset': 0,
4267             'image-pos': 0,
4268             'size': len(data),
4269             }
4270         self.assertEqual(expected, props)
4271
4272     def testCompressExtra(self):
4273         """Test compression of a section with no fixed size"""
4274         self._CheckLz4()
4275         data, _, _, out_dtb_fname = self._DoReadFileDtb(
4276             '186_compress_extra.dts', use_real_dtb=True, update_dtb=True)
4277         dtb = fdt.Fdt(out_dtb_fname)
4278         dtb.Scan()
4279         props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size',
4280                                         'uncomp-size'])
4281
4282         base = data[len(U_BOOT_DATA):]
4283         self.assertEquals(U_BOOT_DATA, base[:len(U_BOOT_DATA)])
4284         rest = base[len(U_BOOT_DATA):]
4285
4286         # Check compressed data
4287         section1 = self._decompress(rest)
4288         expect1 = comp_util.compress(COMPRESS_DATA + U_BOOT_DATA, 'lz4')
4289         self.assertEquals(expect1, rest[:len(expect1)])
4290         self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1)
4291         rest1 = rest[len(expect1):]
4292
4293         section2 = self._decompress(rest1)
4294         expect2 = comp_util.compress(COMPRESS_DATA + COMPRESS_DATA, 'lz4')
4295         self.assertEquals(expect2, rest1[:len(expect2)])
4296         self.assertEquals(COMPRESS_DATA + COMPRESS_DATA, section2)
4297         rest2 = rest1[len(expect2):]
4298
4299         expect_size = (len(U_BOOT_DATA) + len(U_BOOT_DATA) + len(expect1) +
4300                        len(expect2) + len(U_BOOT_DATA))
4301         #self.assertEquals(expect_size, len(data))
4302
4303         #self.assertEquals(U_BOOT_DATA, rest2)
4304
4305         self.maxDiff = None
4306         expected = {
4307             'u-boot:offset': 0,
4308             'u-boot:image-pos': 0,
4309             'u-boot:size': len(U_BOOT_DATA),
4310
4311             'base:offset': len(U_BOOT_DATA),
4312             'base:image-pos': len(U_BOOT_DATA),
4313             'base:size': len(data) - len(U_BOOT_DATA),
4314             'base/u-boot:offset': 0,
4315             'base/u-boot:image-pos': len(U_BOOT_DATA),
4316             'base/u-boot:size': len(U_BOOT_DATA),
4317             'base/u-boot2:offset': len(U_BOOT_DATA) + len(expect1) +
4318                 len(expect2),
4319             'base/u-boot2:image-pos': len(U_BOOT_DATA) * 2 + len(expect1) +
4320                 len(expect2),
4321             'base/u-boot2:size': len(U_BOOT_DATA),
4322
4323             'base/section:offset': len(U_BOOT_DATA),
4324             'base/section:image-pos': len(U_BOOT_DATA) * 2,
4325             'base/section:size': len(expect1),
4326             'base/section:uncomp-size': len(COMPRESS_DATA + U_BOOT_DATA),
4327             'base/section/blob:offset': 0,
4328             'base/section/blob:size': len(COMPRESS_DATA),
4329             'base/section/u-boot:offset': len(COMPRESS_DATA),
4330             'base/section/u-boot:size': len(U_BOOT_DATA),
4331
4332             'base/section2:offset': len(U_BOOT_DATA) + len(expect1),
4333             'base/section2:image-pos': len(U_BOOT_DATA) * 2 + len(expect1),
4334             'base/section2:size': len(expect2),
4335             'base/section2:uncomp-size': len(COMPRESS_DATA + COMPRESS_DATA),
4336             'base/section2/blob:offset': 0,
4337             'base/section2/blob:size': len(COMPRESS_DATA),
4338             'base/section2/blob2:offset': len(COMPRESS_DATA),
4339             'base/section2/blob2:size': len(COMPRESS_DATA),
4340
4341             'offset': 0,
4342             'image-pos': 0,
4343             'size': len(data),
4344             }
4345         self.assertEqual(expected, props)
4346
4347     def testSymbolsSubsection(self):
4348         """Test binman can assign symbols from a subsection"""
4349         self.checkSymbols('187_symbols_sub.dts', U_BOOT_SPL_DATA, 0x18)
4350
4351     def testReadImageEntryArg(self):
4352         """Test reading an image that would need an entry arg to generate"""
4353         entry_args = {
4354             'cros-ec-rw-path': 'ecrw.bin',
4355         }
4356         data = self.data = self._DoReadFileDtb(
4357             '188_image_entryarg.dts',use_real_dtb=True, update_dtb=True,
4358             entry_args=entry_args)
4359
4360         image_fname = tools.get_output_filename('image.bin')
4361         orig_image = control.images['image']
4362
4363         # This should not generate an error about the missing 'cros-ec-rw-path'
4364         # since we are reading the image from a file. Compare with
4365         # testEntryArgsRequired()
4366         image = Image.FromFile(image_fname)
4367         self.assertEqual(orig_image.GetEntries().keys(),
4368                          image.GetEntries().keys())
4369
4370     def testFilesAlign(self):
4371         """Test alignment with files"""
4372         data = self._DoReadFile('190_files_align.dts')
4373
4374         # The first string is 15 bytes so will align to 16
4375         expect = FILES_DATA[:15] + b'\0' + FILES_DATA[15:]
4376         self.assertEqual(expect, data)
4377
4378     def testReadImageSkip(self):
4379         """Test reading an image and accessing its FDT map"""
4380         data = self.data = self._DoReadFileRealDtb('191_read_image_skip.dts')
4381         image_fname = tools.get_output_filename('image.bin')
4382         orig_image = control.images['image']
4383         image = Image.FromFile(image_fname)
4384         self.assertEqual(orig_image.GetEntries().keys(),
4385                          image.GetEntries().keys())
4386
4387         orig_entry = orig_image.GetEntries()['fdtmap']
4388         entry = image.GetEntries()['fdtmap']
4389         self.assertEqual(orig_entry.offset, entry.offset)
4390         self.assertEqual(orig_entry.size, entry.size)
4391         self.assertEqual(16, entry.image_pos)
4392
4393         u_boot = image.GetEntries()['section'].GetEntries()['u-boot']
4394
4395         self.assertEquals(U_BOOT_DATA, u_boot.ReadData())
4396
4397     def testTplNoDtb(self):
4398         """Test that an image with tpl/u-boot-tpl-nodtb.bin can be created"""
4399         self._SetupTplElf()
4400         data = self._DoReadFile('192_u_boot_tpl_nodtb.dts')
4401         self.assertEqual(U_BOOT_TPL_NODTB_DATA,
4402                          data[:len(U_BOOT_TPL_NODTB_DATA)])
4403
4404     def testTplBssPad(self):
4405         """Test that we can pad TPL's BSS with zeros"""
4406         # ELF file with a '__bss_size' symbol
4407         self._SetupTplElf()
4408         data = self._DoReadFile('193_tpl_bss_pad.dts')
4409         self.assertEqual(U_BOOT_TPL_DATA + tools.get_bytes(0, 10) + U_BOOT_DATA,
4410                          data)
4411
4412     def testTplBssPadMissing(self):
4413         """Test that a missing symbol is detected"""
4414         self._SetupTplElf('u_boot_ucode_ptr')
4415         with self.assertRaises(ValueError) as e:
4416             self._DoReadFile('193_tpl_bss_pad.dts')
4417         self.assertIn('Expected __bss_size symbol in tpl/u-boot-tpl',
4418                       str(e.exception))
4419
4420     def checkDtbSizes(self, data, pad_len, start):
4421         """Check the size arguments in a dtb embedded in an image
4422
4423         Args:
4424             data: The image data
4425             pad_len: Length of the pad section in the image, in bytes
4426             start: Start offset of the devicetree to examine, within the image
4427
4428         Returns:
4429             Size of the devicetree in bytes
4430         """
4431         dtb_data = data[start:]
4432         dtb = fdt.Fdt.FromData(dtb_data)
4433         fdt_size = dtb.GetFdtObj().totalsize()
4434         dtb.Scan()
4435         props = self._GetPropTree(dtb, 'size')
4436         self.assertEqual({
4437             'size': len(data),
4438             'u-boot-spl/u-boot-spl-bss-pad:size': pad_len,
4439             'u-boot-spl/u-boot-spl-dtb:size': 801,
4440             'u-boot-spl/u-boot-spl-nodtb:size': len(U_BOOT_SPL_NODTB_DATA),
4441             'u-boot-spl:size': 860,
4442             'u-boot-tpl:size': len(U_BOOT_TPL_DATA),
4443             'u-boot/u-boot-dtb:size': 781,
4444             'u-boot/u-boot-nodtb:size': len(U_BOOT_NODTB_DATA),
4445             'u-boot:size': 827,
4446             }, props)
4447         return fdt_size
4448
4449     def testExpanded(self):
4450         """Test that an expanded entry type is selected when needed"""
4451         self._SetupSplElf()
4452         self._SetupTplElf()
4453
4454         # SPL has a devicetree, TPL does not
4455         entry_args = {
4456             'spl-dtb': '1',
4457             'spl-bss-pad': 'y',
4458             'tpl-dtb': '',
4459         }
4460         self._DoReadFileDtb('194_fdt_incl.dts', use_expanded=True,
4461                             entry_args=entry_args)
4462         image = control.images['image']
4463         entries = image.GetEntries()
4464         self.assertEqual(3, len(entries))
4465
4466         # First, u-boot, which should be expanded into u-boot-nodtb and dtb
4467         self.assertIn('u-boot', entries)
4468         entry = entries['u-boot']
4469         self.assertEqual('u-boot-expanded', entry.etype)
4470         subent = entry.GetEntries()
4471         self.assertEqual(2, len(subent))
4472         self.assertIn('u-boot-nodtb', subent)
4473         self.assertIn('u-boot-dtb', subent)
4474
4475         # Second, u-boot-spl, which should be expanded into three parts
4476         self.assertIn('u-boot-spl', entries)
4477         entry = entries['u-boot-spl']
4478         self.assertEqual('u-boot-spl-expanded', entry.etype)
4479         subent = entry.GetEntries()
4480         self.assertEqual(3, len(subent))
4481         self.assertIn('u-boot-spl-nodtb', subent)
4482         self.assertIn('u-boot-spl-bss-pad', subent)
4483         self.assertIn('u-boot-spl-dtb', subent)
4484
4485         # Third, u-boot-tpl, which should be not be expanded, since TPL has no
4486         # devicetree
4487         self.assertIn('u-boot-tpl', entries)
4488         entry = entries['u-boot-tpl']
4489         self.assertEqual('u-boot-tpl', entry.etype)
4490         self.assertEqual(None, entry.GetEntries())
4491
4492     def testExpandedTpl(self):
4493         """Test that an expanded entry type is selected for TPL when needed"""
4494         self._SetupTplElf()
4495
4496         entry_args = {
4497             'tpl-bss-pad': 'y',
4498             'tpl-dtb': 'y',
4499         }
4500         self._DoReadFileDtb('195_fdt_incl_tpl.dts', use_expanded=True,
4501                             entry_args=entry_args)
4502         image = control.images['image']
4503         entries = image.GetEntries()
4504         self.assertEqual(1, len(entries))
4505
4506         # We only have u-boot-tpl, which be expanded
4507         self.assertIn('u-boot-tpl', entries)
4508         entry = entries['u-boot-tpl']
4509         self.assertEqual('u-boot-tpl-expanded', entry.etype)
4510         subent = entry.GetEntries()
4511         self.assertEqual(3, len(subent))
4512         self.assertIn('u-boot-tpl-nodtb', subent)
4513         self.assertIn('u-boot-tpl-bss-pad', subent)
4514         self.assertIn('u-boot-tpl-dtb', subent)
4515
4516     def testExpandedNoPad(self):
4517         """Test an expanded entry without BSS pad enabled"""
4518         self._SetupSplElf()
4519         self._SetupTplElf()
4520
4521         # SPL has a devicetree, TPL does not
4522         entry_args = {
4523             'spl-dtb': 'something',
4524             'spl-bss-pad': 'n',
4525             'tpl-dtb': '',
4526         }
4527         self._DoReadFileDtb('194_fdt_incl.dts', use_expanded=True,
4528                             entry_args=entry_args)
4529         image = control.images['image']
4530         entries = image.GetEntries()
4531
4532         # Just check u-boot-spl, which should be expanded into two parts
4533         self.assertIn('u-boot-spl', entries)
4534         entry = entries['u-boot-spl']
4535         self.assertEqual('u-boot-spl-expanded', entry.etype)
4536         subent = entry.GetEntries()
4537         self.assertEqual(2, len(subent))
4538         self.assertIn('u-boot-spl-nodtb', subent)
4539         self.assertIn('u-boot-spl-dtb', subent)
4540
4541     def testExpandedTplNoPad(self):
4542         """Test that an expanded entry type with padding disabled in TPL"""
4543         self._SetupTplElf()
4544
4545         entry_args = {
4546             'tpl-bss-pad': '',
4547             'tpl-dtb': 'y',
4548         }
4549         self._DoReadFileDtb('195_fdt_incl_tpl.dts', use_expanded=True,
4550                             entry_args=entry_args)
4551         image = control.images['image']
4552         entries = image.GetEntries()
4553         self.assertEqual(1, len(entries))
4554
4555         # We only have u-boot-tpl, which be expanded
4556         self.assertIn('u-boot-tpl', entries)
4557         entry = entries['u-boot-tpl']
4558         self.assertEqual('u-boot-tpl-expanded', entry.etype)
4559         subent = entry.GetEntries()
4560         self.assertEqual(2, len(subent))
4561         self.assertIn('u-boot-tpl-nodtb', subent)
4562         self.assertIn('u-boot-tpl-dtb', subent)
4563
4564     def testFdtInclude(self):
4565         """Test that an Fdt is update within all binaries"""
4566         self._SetupSplElf()
4567         self._SetupTplElf()
4568
4569         # SPL has a devicetree, TPL does not
4570         self.maxDiff = None
4571         entry_args = {
4572             'spl-dtb': '1',
4573             'spl-bss-pad': 'y',
4574             'tpl-dtb': '',
4575         }
4576         # Build the image. It includes two separate devicetree binaries, each
4577         # with their own contents, but all contain the binman definition.
4578         data = self._DoReadFileDtb(
4579             '194_fdt_incl.dts', use_real_dtb=True, use_expanded=True,
4580             update_dtb=True, entry_args=entry_args)[0]
4581         pad_len = 10
4582
4583         # Check the U-Boot dtb
4584         start = len(U_BOOT_NODTB_DATA)
4585         fdt_size = self.checkDtbSizes(data, pad_len, start)
4586
4587         # Now check SPL
4588         start += fdt_size + len(U_BOOT_SPL_NODTB_DATA) + pad_len
4589         fdt_size = self.checkDtbSizes(data, pad_len, start)
4590
4591         # TPL has no devicetree
4592         start += fdt_size + len(U_BOOT_TPL_DATA)
4593         self.assertEqual(len(data), start)
4594
4595     def testSymbolsExpanded(self):
4596         """Test binman can assign symbols in expanded entries"""
4597         entry_args = {
4598             'spl-dtb': '1',
4599         }
4600         self.checkSymbols('197_symbols_expand.dts', U_BOOT_SPL_NODTB_DATA +
4601                           U_BOOT_SPL_DTB_DATA, 0x38,
4602                           entry_args=entry_args, use_expanded=True)
4603
4604     def testCollection(self):
4605         """Test a collection"""
4606         data = self._DoReadFile('198_collection.dts')
4607         self.assertEqual(U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA +
4608                          tools.get_bytes(0xff, 2) + U_BOOT_NODTB_DATA +
4609                          tools.get_bytes(0xfe, 3) + U_BOOT_DTB_DATA,
4610                          data)
4611
4612     def testCollectionSection(self):
4613         """Test a collection where a section must be built first"""
4614         # Sections never have their contents when GetData() is called, but when
4615         # BuildSectionData() is called with required=True, a section will force
4616         # building the contents, producing an error is anything is still
4617         # missing.
4618         data = self._DoReadFile('199_collection_section.dts')
4619         section = U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA
4620         self.assertEqual(section + U_BOOT_DATA + tools.get_bytes(0xff, 2) +
4621                          section + tools.get_bytes(0xfe, 3) + U_BOOT_DATA,
4622                          data)
4623
4624     def testAlignDefault(self):
4625         """Test that default alignment works on sections"""
4626         data = self._DoReadFile('200_align_default.dts')
4627         expected = (U_BOOT_DATA + tools.get_bytes(0, 8 - len(U_BOOT_DATA)) +
4628                     U_BOOT_DATA)
4629         # Special alignment for section
4630         expected += tools.get_bytes(0, 32 - len(expected))
4631         # No alignment within the nested section
4632         expected += U_BOOT_DATA + U_BOOT_NODTB_DATA;
4633         # Now the final piece, which should be default-aligned
4634         expected += tools.get_bytes(0, 88 - len(expected)) + U_BOOT_NODTB_DATA
4635         self.assertEqual(expected, data)
4636
4637     def testPackOpenSBI(self):
4638         """Test that an image with an OpenSBI binary can be created"""
4639         data = self._DoReadFile('201_opensbi.dts')
4640         self.assertEqual(OPENSBI_DATA, data[:len(OPENSBI_DATA)])
4641
4642     def testSectionsSingleThread(self):
4643         """Test sections without multithreading"""
4644         data = self._DoReadFileDtb('055_sections.dts', threads=0)[0]
4645         expected = (U_BOOT_DATA + tools.get_bytes(ord('!'), 12) +
4646                     U_BOOT_DATA + tools.get_bytes(ord('a'), 12) +
4647                     U_BOOT_DATA + tools.get_bytes(ord('&'), 4))
4648         self.assertEqual(expected, data)
4649
4650     def testThreadTimeout(self):
4651         """Test handling a thread that takes too long"""
4652         with self.assertRaises(ValueError) as e:
4653             self._DoTestFile('202_section_timeout.dts',
4654                              test_section_timeout=True)
4655         self.assertIn("Timed out obtaining contents", str(e.exception))
4656
4657     def testTiming(self):
4658         """Test output of timing information"""
4659         data = self._DoReadFile('055_sections.dts')
4660         with test_util.capture_sys_output() as (stdout, stderr):
4661             state.TimingShow()
4662         self.assertIn('read:', stdout.getvalue())
4663         self.assertIn('compress:', stdout.getvalue())
4664
4665     def testUpdateFdtInElf(self):
4666         """Test that we can update the devicetree in an ELF file"""
4667         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed')
4668         outfile = os.path.join(self._indir, 'u-boot.out')
4669         begin_sym = 'dtb_embed_begin'
4670         end_sym = 'dtb_embed_end'
4671         retcode = self._DoTestFile(
4672             '060_fdt_update.dts', update_dtb=True,
4673             update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4674         self.assertEqual(0, retcode)
4675
4676         # Check that the output file does in fact contact a dtb with the binman
4677         # definition in the correct place
4678         syms = elf.GetSymbolFileOffset(infile,
4679                                        ['dtb_embed_begin', 'dtb_embed_end'])
4680         data = tools.read_file(outfile)
4681         dtb_data = data[syms['dtb_embed_begin'].offset:
4682                         syms['dtb_embed_end'].offset]
4683
4684         dtb = fdt.Fdt.FromData(dtb_data)
4685         dtb.Scan()
4686         props = self._GetPropTree(dtb, BASE_DTB_PROPS + REPACK_DTB_PROPS)
4687         self.assertEqual({
4688             'image-pos': 0,
4689             'offset': 0,
4690             '_testing:offset': 32,
4691             '_testing:size': 2,
4692             '_testing:image-pos': 32,
4693             'section@0/u-boot:offset': 0,
4694             'section@0/u-boot:size': len(U_BOOT_DATA),
4695             'section@0/u-boot:image-pos': 0,
4696             'section@0:offset': 0,
4697             'section@0:size': 16,
4698             'section@0:image-pos': 0,
4699
4700             'section@1/u-boot:offset': 0,
4701             'section@1/u-boot:size': len(U_BOOT_DATA),
4702             'section@1/u-boot:image-pos': 16,
4703             'section@1:offset': 16,
4704             'section@1:size': 16,
4705             'section@1:image-pos': 16,
4706             'size': 40
4707         }, props)
4708
4709     def testUpdateFdtInElfInvalid(self):
4710         """Test that invalid args are detected with --update-fdt-in-elf"""
4711         with self.assertRaises(ValueError) as e:
4712             self._DoTestFile('060_fdt_update.dts', update_fdt_in_elf='fred')
4713         self.assertIn("Invalid args ['fred'] to --update-fdt-in-elf",
4714                       str(e.exception))
4715
4716     def testUpdateFdtInElfNoSyms(self):
4717         """Test that missing symbols are detected with --update-fdt-in-elf"""
4718         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed')
4719         outfile = ''
4720         begin_sym = 'wrong_begin'
4721         end_sym = 'wrong_end'
4722         with self.assertRaises(ValueError) as e:
4723             self._DoTestFile(
4724                 '060_fdt_update.dts',
4725                 update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4726         self.assertIn("Expected two symbols 'wrong_begin' and 'wrong_end': got 0:",
4727                       str(e.exception))
4728
4729     def testUpdateFdtInElfTooSmall(self):
4730         """Test that an over-large dtb is detected with --update-fdt-in-elf"""
4731         infile = elf_fname = self.ElfTestFile('u_boot_binman_embed_sm')
4732         outfile = os.path.join(self._indir, 'u-boot.out')
4733         begin_sym = 'dtb_embed_begin'
4734         end_sym = 'dtb_embed_end'
4735         with self.assertRaises(ValueError) as e:
4736             self._DoTestFile(
4737                 '060_fdt_update.dts', update_dtb=True,
4738                 update_fdt_in_elf=','.join([infile,outfile,begin_sym,end_sym]))
4739         self.assertRegex(
4740             str(e.exception),
4741             "Not enough space in '.*u_boot_binman_embed_sm' for data length.*")
4742
4743     def testVersion(self):
4744         """Test we can get the binman version"""
4745         version = '(unreleased)'
4746         self.assertEqual(version, state.GetVersion(self._indir))
4747
4748         with self.assertRaises(SystemExit):
4749             with test_util.capture_sys_output() as (_, stderr):
4750                 self._DoBinman('-V')
4751         self.assertEqual('Binman %s\n' % version, stderr.getvalue())
4752
4753         # Try running the tool too, just to be safe
4754         result = self._RunBinman('-V')
4755         self.assertEqual('Binman %s\n' % version, result.stderr)
4756
4757         # Set up a version file to make sure that works
4758         version = 'v2025.01-rc2'
4759         tools.write_file(os.path.join(self._indir, 'version'), version,
4760                         binary=False)
4761         self.assertEqual(version, state.GetVersion(self._indir))
4762
4763     def testAltFormat(self):
4764         """Test that alternative formats can be used to extract"""
4765         self._DoReadFileRealDtb('213_fdtmap_alt_format.dts')
4766
4767         try:
4768             tmpdir, updated_fname = self._SetupImageInTmpdir()
4769             with test_util.capture_sys_output() as (stdout, _):
4770                 self._DoBinman('extract', '-i', updated_fname, '-F', 'list')
4771             self.assertEqual(
4772                 '''Flag (-F)   Entry type            Description
4773 fdt         fdtmap                Extract the devicetree blob from the fdtmap
4774 ''',
4775                 stdout.getvalue())
4776
4777             dtb = os.path.join(tmpdir, 'fdt.dtb')
4778             self._DoBinman('extract', '-i', updated_fname, '-F', 'fdt', '-f',
4779                            dtb, 'fdtmap')
4780
4781             # Check that we can read it and it can be scanning, meaning it does
4782             # not have a 16-byte fdtmap header
4783             data = tools.read_file(dtb)
4784             dtb = fdt.Fdt.FromData(data)
4785             dtb.Scan()
4786
4787             # Now check u-boot which has no alt_format
4788             fname = os.path.join(tmpdir, 'fdt.dtb')
4789             self._DoBinman('extract', '-i', updated_fname, '-F', 'dummy',
4790                            '-f', fname, 'u-boot')
4791             data = tools.read_file(fname)
4792             self.assertEqual(U_BOOT_DATA, data)
4793
4794         finally:
4795             shutil.rmtree(tmpdir)
4796
4797     def testExtblobList(self):
4798         """Test an image with an external blob list"""
4799         data = self._DoReadFile('215_blob_ext_list.dts')
4800         self.assertEqual(REFCODE_DATA + FSP_M_DATA, data)
4801
4802     def testExtblobListMissing(self):
4803         """Test an image with a missing external blob"""
4804         with self.assertRaises(ValueError) as e:
4805             self._DoReadFile('216_blob_ext_list_missing.dts')
4806         self.assertIn("Filename 'missing-file' not found in input path",
4807                       str(e.exception))
4808
4809     def testExtblobListMissingOk(self):
4810         """Test an image with an missing external blob that is allowed"""
4811         with test_util.capture_sys_output() as (stdout, stderr):
4812             self._DoTestFile('216_blob_ext_list_missing.dts',
4813                              allow_missing=True)
4814         err = stderr.getvalue()
4815         self.assertRegex(err, "Image 'main-section'.*missing.*: blob-ext")
4816
4817     def testFip(self):
4818         """Basic test of generation of an ARM Firmware Image Package (FIP)"""
4819         data = self._DoReadFile('203_fip.dts')
4820         hdr, fents = fip_util.decode_fip(data)
4821         self.assertEqual(fip_util.HEADER_MAGIC, hdr.name)
4822         self.assertEqual(fip_util.HEADER_SERIAL, hdr.serial)
4823         self.assertEqual(0x123, hdr.flags)
4824
4825         self.assertEqual(2, len(fents))
4826
4827         fent = fents[0]
4828         self.assertEqual(
4829             bytes([0x47,  0xd4, 0x08, 0x6d, 0x4c, 0xfe, 0x98, 0x46,
4830                   0x9b, 0x95, 0x29, 0x50, 0xcb, 0xbd, 0x5a, 0x0]), fent.uuid)
4831         self.assertEqual('soc-fw', fent.fip_type)
4832         self.assertEqual(0x88, fent.offset)
4833         self.assertEqual(len(ATF_BL31_DATA), fent.size)
4834         self.assertEqual(0x123456789abcdef, fent.flags)
4835         self.assertEqual(ATF_BL31_DATA, fent.data)
4836         self.assertEqual(True, fent.valid)
4837
4838         fent = fents[1]
4839         self.assertEqual(
4840             bytes([0x65, 0x92, 0x27, 0x03, 0x2f, 0x74, 0xe6, 0x44,
4841              0x8d, 0xff, 0x57, 0x9a, 0xc1, 0xff, 0x06, 0x10]), fent.uuid)
4842         self.assertEqual('scp-fwu-cfg', fent.fip_type)
4843         self.assertEqual(0x8c, fent.offset)
4844         self.assertEqual(len(ATF_BL31_DATA), fent.size)
4845         self.assertEqual(0, fent.flags)
4846         self.assertEqual(ATF_BL2U_DATA, fent.data)
4847         self.assertEqual(True, fent.valid)
4848
4849     def testFipOther(self):
4850         """Basic FIP with something that isn't a external blob"""
4851         data = self._DoReadFile('204_fip_other.dts')
4852         hdr, fents = fip_util.decode_fip(data)
4853
4854         self.assertEqual(2, len(fents))
4855         fent = fents[1]
4856         self.assertEqual('rot-cert', fent.fip_type)
4857         self.assertEqual(b'aa', fent.data)
4858
4859     def testFipOther(self):
4860         """Basic FIP with something that isn't a external blob"""
4861         data = self._DoReadFile('204_fip_other.dts')
4862         hdr, fents = fip_util.decode_fip(data)
4863
4864         self.assertEqual(2, len(fents))
4865         fent = fents[1]
4866         self.assertEqual('rot-cert', fent.fip_type)
4867         self.assertEqual(b'aa', fent.data)
4868
4869     def testFipNoType(self):
4870         """FIP with an entry of an unknown type"""
4871         with self.assertRaises(ValueError) as e:
4872             self._DoReadFile('205_fip_no_type.dts')
4873         self.assertIn("Must provide a fip-type (node name 'u-boot' is not a known FIP type)",
4874                       str(e.exception))
4875
4876     def testFipUuid(self):
4877         """Basic FIP with a manual uuid"""
4878         data = self._DoReadFile('206_fip_uuid.dts')
4879         hdr, fents = fip_util.decode_fip(data)
4880
4881         self.assertEqual(2, len(fents))
4882         fent = fents[1]
4883         self.assertEqual(None, fent.fip_type)
4884         self.assertEqual(
4885             bytes([0xfc, 0x65, 0x13, 0x92, 0x4a, 0x5b, 0x11, 0xec,
4886                    0x94, 0x35, 0xff, 0x2d, 0x1c, 0xfc, 0x79, 0x9c]),
4887             fent.uuid)
4888         self.assertEqual(U_BOOT_DATA, fent.data)
4889
4890     def testFipLs(self):
4891         """Test listing a FIP"""
4892         data = self._DoReadFileRealDtb('207_fip_ls.dts')
4893         hdr, fents = fip_util.decode_fip(data)
4894
4895         try:
4896             tmpdir, updated_fname = self._SetupImageInTmpdir()
4897             with test_util.capture_sys_output() as (stdout, stderr):
4898                 self._DoBinman('ls', '-i', updated_fname)
4899         finally:
4900             shutil.rmtree(tmpdir)
4901         lines = stdout.getvalue().splitlines()
4902         expected = [
4903 'Name          Image-pos  Size  Entry-type  Offset  Uncomp-size',
4904 '----------------------------------------------------------------',
4905 'main-section          0   2d3  section          0',
4906 '  atf-fip             0    90  atf-fip          0',
4907 '    soc-fw           88     4  blob-ext        88',
4908 '    u-boot           8c     4  u-boot          8c',
4909 '  fdtmap             90   243  fdtmap          90',
4910 ]
4911         self.assertEqual(expected, lines)
4912
4913         image = control.images['image']
4914         entries = image.GetEntries()
4915         fdtmap = entries['fdtmap']
4916
4917         fdtmap_data = data[fdtmap.image_pos:fdtmap.image_pos + fdtmap.size]
4918         magic = fdtmap_data[:8]
4919         self.assertEqual(b'_FDTMAP_', magic)
4920         self.assertEqual(tools.get_bytes(0, 8), fdtmap_data[8:16])
4921
4922         fdt_data = fdtmap_data[16:]
4923         dtb = fdt.Fdt.FromData(fdt_data)
4924         dtb.Scan()
4925         props = self._GetPropTree(dtb, BASE_DTB_PROPS, prefix='/')
4926         self.assertEqual({
4927             'atf-fip/soc-fw:image-pos': 136,
4928             'atf-fip/soc-fw:offset': 136,
4929             'atf-fip/soc-fw:size': 4,
4930             'atf-fip/u-boot:image-pos': 140,
4931             'atf-fip/u-boot:offset': 140,
4932             'atf-fip/u-boot:size': 4,
4933             'atf-fip:image-pos': 0,
4934             'atf-fip:offset': 0,
4935             'atf-fip:size': 144,
4936             'image-pos': 0,
4937             'offset': 0,
4938             'fdtmap:image-pos': fdtmap.image_pos,
4939             'fdtmap:offset': fdtmap.offset,
4940             'fdtmap:size': len(fdtmap_data),
4941             'size': len(data),
4942         }, props)
4943
4944     def testFipExtractOneEntry(self):
4945         """Test extracting a single entry fron an FIP"""
4946         self._DoReadFileRealDtb('207_fip_ls.dts')
4947         image_fname = tools.get_output_filename('image.bin')
4948         fname = os.path.join(self._indir, 'output.extact')
4949         control.ExtractEntries(image_fname, fname, None, ['atf-fip/u-boot'])
4950         data = tools.read_file(fname)
4951         self.assertEqual(U_BOOT_DATA, data)
4952
4953     def testFipReplace(self):
4954         """Test replacing a single file in a FIP"""
4955         expected = U_BOOT_DATA + tools.get_bytes(0x78, 50)
4956         data = self._DoReadFileRealDtb('208_fip_replace.dts')
4957         updated_fname = tools.get_output_filename('image-updated.bin')
4958         tools.write_file(updated_fname, data)
4959         entry_name = 'atf-fip/u-boot'
4960         control.WriteEntry(updated_fname, entry_name, expected,
4961                            allow_resize=True)
4962         actual = control.ReadEntry(updated_fname, entry_name)
4963         self.assertEqual(expected, actual)
4964
4965         new_data = tools.read_file(updated_fname)
4966         hdr, fents = fip_util.decode_fip(new_data)
4967
4968         self.assertEqual(2, len(fents))
4969
4970         # Check that the FIP entry is updated
4971         fent = fents[1]
4972         self.assertEqual(0x8c, fent.offset)
4973         self.assertEqual(len(expected), fent.size)
4974         self.assertEqual(0, fent.flags)
4975         self.assertEqual(expected, fent.data)
4976         self.assertEqual(True, fent.valid)
4977
4978     def testFipMissing(self):
4979         with test_util.capture_sys_output() as (stdout, stderr):
4980             self._DoTestFile('209_fip_missing.dts', allow_missing=True)
4981         err = stderr.getvalue()
4982         self.assertRegex(err, "Image 'main-section'.*missing.*: rmm-fw")
4983
4984     def testFipSize(self):
4985         """Test a FIP with a size property"""
4986         data = self._DoReadFile('210_fip_size.dts')
4987         self.assertEqual(0x100 + len(U_BOOT_DATA), len(data))
4988         hdr, fents = fip_util.decode_fip(data)
4989         self.assertEqual(fip_util.HEADER_MAGIC, hdr.name)
4990         self.assertEqual(fip_util.HEADER_SERIAL, hdr.serial)
4991
4992         self.assertEqual(1, len(fents))
4993
4994         fent = fents[0]
4995         self.assertEqual('soc-fw', fent.fip_type)
4996         self.assertEqual(0x60, fent.offset)
4997         self.assertEqual(len(ATF_BL31_DATA), fent.size)
4998         self.assertEqual(ATF_BL31_DATA, fent.data)
4999         self.assertEqual(True, fent.valid)
5000
5001         rest = data[0x60 + len(ATF_BL31_DATA):0x100]
5002         self.assertEqual(tools.get_bytes(0xff, len(rest)), rest)
5003
5004     def testFipBadAlign(self):
5005         """Test that an invalid alignment value in a FIP is detected"""
5006         with self.assertRaises(ValueError) as e:
5007             self._DoTestFile('211_fip_bad_align.dts')
5008         self.assertIn(
5009             "Node \'/binman/atf-fip\': FIP alignment 31 must be a power of two",
5010             str(e.exception))
5011
5012     def testFipCollection(self):
5013         """Test using a FIP in a collection"""
5014         data = self._DoReadFile('212_fip_collection.dts')
5015         entry1 = control.images['image'].GetEntries()['collection']
5016         data1 = data[:entry1.size]
5017         hdr1, fents2 = fip_util.decode_fip(data1)
5018
5019         entry2 = control.images['image'].GetEntries()['atf-fip']
5020         data2 = data[entry2.offset:entry2.offset + entry2.size]
5021         hdr1, fents2 = fip_util.decode_fip(data2)
5022
5023         # The 'collection' entry should have U-Boot included at the end
5024         self.assertEqual(entry1.size - len(U_BOOT_DATA), entry2.size)
5025         self.assertEqual(data1, data2 + U_BOOT_DATA)
5026         self.assertEqual(U_BOOT_DATA, data1[-4:])
5027
5028         # There should be a U-Boot after the final FIP
5029         self.assertEqual(U_BOOT_DATA, data[-4:])
5030
5031     def testFakeBlob(self):
5032         """Test handling of faking an external blob"""
5033         with test_util.capture_sys_output() as (stdout, stderr):
5034             self._DoTestFile('217_fake_blob.dts', allow_missing=True,
5035                              allow_fake_blobs=True)
5036         err = stderr.getvalue()
5037         self.assertRegex(
5038             err,
5039             "Image '.*' has faked external blobs and is non-functional: .*")
5040
5041     def testExtblobListFaked(self):
5042         """Test an extblob with missing external blob that are faked"""
5043         with test_util.capture_sys_output() as (stdout, stderr):
5044             self._DoTestFile('216_blob_ext_list_missing.dts',
5045                              allow_fake_blobs=True)
5046         err = stderr.getvalue()
5047         self.assertRegex(err, "Image 'main-section'.*faked.*: blob-ext-list")
5048
5049     def testListBintools(self):
5050         args = ['tool', '--list']
5051         with test_util.capture_sys_output() as (stdout, _):
5052             self._DoBinman(*args)
5053         out = stdout.getvalue().splitlines()
5054         self.assertTrue(len(out) >= 2)
5055
5056     def testFetchBintools(self):
5057         def fail_download(url):
5058             """Take the tools.download() function by raising an exception"""
5059             raise urllib.error.URLError('my error')
5060
5061         args = ['tool']
5062         with self.assertRaises(ValueError) as e:
5063             self._DoBinman(*args)
5064         self.assertIn("Invalid arguments to 'tool' subcommand",
5065                       str(e.exception))
5066
5067         args = ['tool', '--fetch']
5068         with self.assertRaises(ValueError) as e:
5069             self._DoBinman(*args)
5070         self.assertIn('Please specify bintools to fetch', str(e.exception))
5071
5072         args = ['tool', '--fetch', '_testing']
5073         with unittest.mock.patch.object(tools, 'download',
5074                                         side_effect=fail_download):
5075             with test_util.capture_sys_output() as (stdout, _):
5076                 self._DoBinman(*args)
5077         self.assertIn('failed to fetch with all methods', stdout.getvalue())
5078
5079     def testInvalidCompress(self):
5080         with self.assertRaises(ValueError) as e:
5081             comp_util.compress(b'', 'invalid')
5082         self.assertIn("Unknown algorithm 'invalid'", str(e.exception))
5083
5084         with self.assertRaises(ValueError) as e:
5085             comp_util.decompress(b'1234', 'invalid')
5086         self.assertIn("Unknown algorithm 'invalid'", str(e.exception))
5087
5088     def testBintoolDocs(self):
5089         """Test for creation of bintool documentation"""
5090         with test_util.capture_sys_output() as (stdout, stderr):
5091             control.write_bintool_docs(control.bintool.Bintool.get_tool_list())
5092         self.assertTrue(len(stdout.getvalue()) > 0)
5093
5094     def testBintoolDocsMissing(self):
5095         """Test handling of missing bintool documentation"""
5096         with self.assertRaises(ValueError) as e:
5097             with test_util.capture_sys_output() as (stdout, stderr):
5098                 control.write_bintool_docs(
5099                     control.bintool.Bintool.get_tool_list(), 'mkimage')
5100         self.assertIn('Documentation is missing for modules: mkimage',
5101                       str(e.exception))
5102
5103     def testListWithGenNode(self):
5104         """Check handling of an FDT map when the section cannot be found"""
5105         entry_args = {
5106             'of-list': 'test-fdt1 test-fdt2',
5107         }
5108         data = self._DoReadFileDtb(
5109             '219_fit_gennode.dts',
5110             entry_args=entry_args,
5111             use_real_dtb=True,
5112             extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])
5113
5114         try:
5115             tmpdir, updated_fname = self._SetupImageInTmpdir()
5116             with test_util.capture_sys_output() as (stdout, stderr):
5117                 self._RunBinman('ls', '-i', updated_fname)
5118         finally:
5119             shutil.rmtree(tmpdir)
5120
5121
5122 if __name__ == "__main__":
5123     unittest.main()