1 # SPDX-License-Identifier: GPL-2.0+
2 # Copyright (c) 2016 Google, Inc
3 # Written by Simon Glass <sjg@chromium.org>
5 # Creates binary images from input files controlled by a description
8 from collections import OrderedDict
15 from patman import tools
17 from binman import cbfs_util
18 from binman import elf
19 from patman import command
20 from patman import tout
22 # List of images we plan to create
23 # Make this global so that it can be referenced from tests
24 images = OrderedDict()
26 # Help text for each type of missing blob, dict:
27 # key: Value of the entry's 'missing-msg' or entry name
28 # value: Text for the help
29 missing_blob_help = {}
31 def _ReadImageDesc(binman_node):
32 """Read the image descriptions from the /binman node
34 This normally produces a single Image object called 'image'. But if
35 multiple images are present, they will all be returned.
38 binman_node: Node object of the /binman node
40 OrderedDict of Image objects, each of which describes an image
42 images = OrderedDict()
43 if 'multiple-images' in binman_node.props:
44 for node in binman_node.subnodes:
45 images[node.name] = Image(node.name, node)
47 images['image'] = Image('image', binman_node)
50 def _FindBinmanNode(dtb):
51 """Find the 'binman' node in the device tree
54 dtb: Fdt object to scan
56 Node object of /binman node, or None if not found
58 for node in dtb.GetRoot().subnodes:
59 if node.name == 'binman':
63 def _ReadMissingBlobHelp():
64 """Read the missing-blob-help file
66 This file containins help messages explaining what to do when external blobs
71 key: Message tag (str)
72 value: Message text (str)
75 def _FinishTag(tag, msg, result):
77 result[tag] = msg.rstrip()
82 my_data = pkg_resources.resource_string(__name__, 'missing-blob-help')
83 re_tag = re.compile('^([-a-z0-9]+):$')
87 for line in my_data.decode('utf-8').splitlines():
88 if not line.startswith('#'):
89 m_tag = re_tag.match(line)
91 _, msg = _FinishTag(tag, msg, result)
95 _FinishTag(tag, msg, result)
98 def _ShowBlobHelp(path, text):
99 tout.Warning('\n%s:' % path)
100 for line in text.splitlines():
101 tout.Warning(' %s' % line)
103 def _ShowHelpForMissingBlobs(missing_list):
104 """Show help for each missing blob to help the user take action
107 missing_list: List of Entry objects to show help for
109 global missing_blob_help
111 if not missing_blob_help:
112 missing_blob_help = _ReadMissingBlobHelp()
114 for entry in missing_list:
115 tags = entry.GetHelpTags()
117 # Show the first match help message
119 if tag in missing_blob_help:
120 _ShowBlobHelp(entry._node.path, missing_blob_help[tag])
123 def GetEntryModules(include_testing=True):
124 """Get a set of entry class implementations
127 Set of paths to entry class filenames
129 glob_list = pkg_resources.resource_listdir(__name__, 'etype')
130 glob_list = [fname for fname in glob_list if fname.endswith('.py')]
131 return set([os.path.splitext(os.path.basename(item))[0]
132 for item in glob_list
133 if include_testing or '_testing' not in item])
135 def WriteEntryDocs(modules, test_missing=None):
136 """Write out documentation for all entries
139 modules: List of Module objects to get docs for
140 test_missing: Used for testing only, to force an entry's documeentation
141 to show as missing even if it is present. Should be set to None in
144 from binman.entry import Entry
145 Entry.WriteDocs(modules, test_missing)
148 def ListEntries(image_fname, entry_paths):
149 """List the entries in an image
151 This decodes the supplied image and displays a table of entries from that
152 image, preceded by a header.
155 image_fname: Image filename to process
156 entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*',
159 image = Image.FromFile(image_fname)
161 entries, lines, widths = image.GetListEntries(entry_paths)
163 num_columns = len(widths)
164 for linenum, line in enumerate(lines):
167 print('-' * (sum(widths) + num_columns * 2))
169 for i, item in enumerate(line):
171 if item.startswith('>'):
174 txt = '%*s ' % (width, item)
179 def ReadEntry(image_fname, entry_path, decomp=True):
180 """Extract an entry from an image
182 This extracts the data from a particular entry in an image
185 image_fname: Image filename to process
186 entry_path: Path to entry to extract
187 decomp: True to return uncompressed data, if the data is compress
188 False to return the raw data
191 data extracted from the entry
194 from binman.image import Image
196 image = Image.FromFile(image_fname)
197 entry = image.FindEntryPath(entry_path)
198 return entry.ReadData(decomp)
201 def ExtractEntries(image_fname, output_fname, outdir, entry_paths,
203 """Extract the data from one or more entries and write it to files
206 image_fname: Image filename to process
207 output_fname: Single output filename to use if extracting one file, None
209 outdir: Output directory to use (for any number of files), else None
210 entry_paths: List of entry paths to extract
211 decomp: True to decompress the entry data
214 List of EntryInfo records that were written
216 image = Image.FromFile(image_fname)
218 # Output an entry to a single file, as a special case
221 raise ValueError('Must specify an entry path to write with -f')
222 if len(entry_paths) != 1:
223 raise ValueError('Must specify exactly one entry path to write with -f')
224 entry = image.FindEntryPath(entry_paths[0])
225 data = entry.ReadData(decomp)
226 tools.WriteFile(output_fname, data)
227 tout.Notice("Wrote %#x bytes to file '%s'" % (len(data), output_fname))
230 # Otherwise we will output to a path given by the entry path of each entry.
231 # This means that entries will appear in subdirectories if they are part of
233 einfos = image.GetListEntries(entry_paths)[0]
234 tout.Notice('%d entries match and will be written' % len(einfos))
237 data = entry.ReadData(decomp)
238 path = entry.GetPath()[1:]
239 fname = os.path.join(outdir, path)
241 # If this entry has children, create a directory for it and put its
242 # data in a file called 'root' in that directory
243 if entry.GetEntries():
244 if not os.path.exists(fname):
246 fname = os.path.join(fname, 'root')
247 tout.Notice("Write entry '%s' to '%s'" % (entry.GetPath(), fname))
248 tools.WriteFile(fname, data)
252 def BeforeReplace(image, allow_resize):
253 """Handle getting an image ready for replacing entries in it
256 image: Image to prepare
258 state.PrepareFromLoadedData(image)
261 # If repacking, drop the old offset/size values except for the original
262 # ones, so we are only left with the constraints.
267 def ReplaceOneEntry(image, entry, data, do_compress, allow_resize):
268 """Handle replacing a single entry an an image
271 image: Image to update
272 entry: Entry to write
273 data: Data to replace with
274 do_compress: True to compress the data if needed, False if data is
275 already compressed so should be used as is
276 allow_resize: True to allow entries to change size (this does a re-pack
277 of the entries), False to raise an exception
279 if not entry.WriteData(data, do_compress):
280 if not image.allow_repack:
281 entry.Raise('Entry data size does not match, but allow-repack is not present for this image')
283 entry.Raise('Entry data size does not match, but resize is disabled')
286 def AfterReplace(image, allow_resize, write_map):
287 """Handle write out an image after replacing entries in it
290 image: Image to write
291 allow_resize: True to allow entries to change size (this does a re-pack
292 of the entries), False to raise an exception
293 write_map: True to write a map file
295 tout.Info('Processing image')
296 ProcessImage(image, update_fdt=True, write_map=write_map,
297 get_contents=False, allow_resize=allow_resize)
300 def WriteEntryToImage(image, entry, data, do_compress=True, allow_resize=True,
302 BeforeReplace(image, allow_resize)
303 tout.Info('Writing data to %s' % entry.GetPath())
304 ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
305 AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
308 def WriteEntry(image_fname, entry_path, data, do_compress=True,
309 allow_resize=True, write_map=False):
310 """Replace an entry in an image
312 This replaces the data in a particular entry in an image. This size of the
313 new data must match the size of the old data unless allow_resize is True.
316 image_fname: Image filename to process
317 entry_path: Path to entry to extract
318 data: Data to replace with
319 do_compress: True to compress the data if needed, False if data is
320 already compressed so should be used as is
321 allow_resize: True to allow entries to change size (this does a re-pack
322 of the entries), False to raise an exception
323 write_map: True to write a map file
326 Image object that was updated
328 tout.Info("Write entry '%s', file '%s'" % (entry_path, image_fname))
329 image = Image.FromFile(image_fname)
330 entry = image.FindEntryPath(entry_path)
331 WriteEntryToImage(image, entry, data, do_compress=do_compress,
332 allow_resize=allow_resize, write_map=write_map)
337 def ReplaceEntries(image_fname, input_fname, indir, entry_paths,
338 do_compress=True, allow_resize=True, write_map=False):
339 """Replace the data from one or more entries from input files
342 image_fname: Image filename to process
343 input_fname: Single input ilename to use if replacing one file, None
345 indir: Input directory to use (for any number of files), else None
346 entry_paths: List of entry paths to extract
347 do_compress: True if the input data is uncompressed and may need to be
348 compressed if the entry requires it, False if the data is already
350 write_map: True to write a map file
353 List of EntryInfo records that were written
355 image = Image.FromFile(image_fname)
357 # Replace an entry from a single file, as a special case
360 raise ValueError('Must specify an entry path to read with -f')
361 if len(entry_paths) != 1:
362 raise ValueError('Must specify exactly one entry path to write with -f')
363 entry = image.FindEntryPath(entry_paths[0])
364 data = tools.ReadFile(input_fname)
365 tout.Notice("Read %#x bytes from file '%s'" % (len(data), input_fname))
366 WriteEntryToImage(image, entry, data, do_compress=do_compress,
367 allow_resize=allow_resize, write_map=write_map)
370 # Otherwise we will input from a path given by the entry path of each entry.
371 # This means that files must appear in subdirectories if they are part of
373 einfos = image.GetListEntries(entry_paths)[0]
374 tout.Notice("Replacing %d matching entries in image '%s'" %
375 (len(einfos), image_fname))
377 BeforeReplace(image, allow_resize)
381 if entry.GetEntries():
382 tout.Info("Skipping section entry '%s'" % entry.GetPath())
385 path = entry.GetPath()[1:]
386 fname = os.path.join(indir, path)
388 if os.path.exists(fname):
389 tout.Notice("Write entry '%s' from file '%s'" %
390 (entry.GetPath(), fname))
391 data = tools.ReadFile(fname)
392 ReplaceOneEntry(image, entry, data, do_compress, allow_resize)
394 tout.Warning("Skipping entry '%s' from missing file '%s'" %
395 (entry.GetPath(), fname))
397 AfterReplace(image, allow_resize=allow_resize, write_map=write_map)
401 def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt):
402 """Prepare the images to be processed and select the device tree
405 - reads in the device tree
406 - finds and scans the binman node to create all entries
407 - selects which images to build
408 - Updates the device tress with placeholder properties for offset,
412 dtb_fname: Filename of the device tree file to use (.dts or .dtb)
413 selected_images: List of images to output, or None for all
414 update_fdt: True to update the FDT wth entry offsets, etc.
417 OrderedDict of images:
418 key: Image name (str)
421 # Import these here in case libfdt.py is not available, in which case
422 # the above help option still works.
424 from dtoc import fdt_util
427 # Get the device tree ready by compiling it and copying the compiled
428 # output into a file in our output directly. Then scan it for use
430 dtb_fname = fdt_util.EnsureCompiled(dtb_fname)
431 fname = tools.GetOutputFilename('u-boot.dtb.out')
432 tools.WriteFile(fname, tools.ReadFile(dtb_fname))
433 dtb = fdt.FdtScan(fname)
435 node = _FindBinmanNode(dtb)
437 raise ValueError("Device tree '%s' does not have a 'binman' "
440 images = _ReadImageDesc(node)
444 new_images = OrderedDict()
445 for name, image in images.items():
446 if name in select_images:
447 new_images[name] = image
451 tout.Notice('Skipping images: %s' % ', '.join(skip))
453 state.Prepare(images, dtb)
455 # Prepare the device tree by making sure that any missing
456 # properties are added (e.g. 'pos' and 'size'). The values of these
457 # may not be correct yet, but we add placeholders so that the
458 # size of the device tree is correct. Later, in
459 # SetCalculatedProperties() we will insert the correct values
460 # without changing the device-tree size, thus ensuring that our
461 # entry offsets remain the same.
462 for image in images.values():
463 image.ExpandEntries()
465 image.AddMissingProperties(True)
466 image.ProcessFdt(dtb)
468 for dtb_item in state.GetAllFdts():
469 dtb_item.Sync(auto_resize=True)
475 def ProcessImage(image, update_fdt, write_map, get_contents=True,
476 allow_resize=True, allow_missing=False):
477 """Perform all steps for this image, including checking and # writing it.
479 This means that errors found with a later image will be reported after
480 earlier images are already completed and written, but that does not seem
484 image: Image to process
485 update_fdt: True to update the FDT wth entry offsets, etc.
486 write_map: True to write a map file
487 get_contents: True to get the image contents from files, etc., False if
488 the contents is already present
489 allow_resize: True to allow entries to change size (this does a re-pack
490 of the entries), False to raise an exception
491 allow_missing: Allow blob_ext objects to be missing
494 True if one or more external blobs are missing, False if all are present
497 image.SetAllowMissing(allow_missing)
498 image.GetEntryContents()
499 image.GetEntryOffsets()
501 # We need to pack the entries to figure out where everything
502 # should be placed. This sets the offset/size of each entry.
503 # However, after packing we call ProcessEntryContents() which
504 # may result in an entry changing size. In that case we need to
505 # do another pass. Since the device tree often contains the
506 # final offset/size information we try to make space for this in
507 # AddMissingProperties() above. However, if the device is
508 # compressed we cannot know this compressed size in advance,
509 # since changing an offset from 0x100 to 0x104 (for example) can
510 # alter the compressed size of the device tree. So we need a
511 # third pass for this.
513 for pack_pass in range(passes):
516 except Exception as e:
518 fname = image.WriteMap()
519 print("Wrote map file '%s' to show errors" % fname)
523 image.SetCalculatedProperties()
524 for dtb_item in state.GetAllFdts():
528 sizes_ok = image.ProcessEntryContents()
532 tout.Info('Pack completed after %d pass(es)' % (pack_pass + 1))
534 image.Raise('Entries changed size after packing (tried %s passes)' %
541 image.CheckMissing(missing_list)
543 tout.Warning("Image '%s' is missing external blobs and is non-functional: %s" %
544 (image.name, ' '.join([e.name for e in missing_list])))
545 _ShowHelpForMissingBlobs(missing_list)
546 return bool(missing_list)
550 """The main control code for binman
552 This assumes that help and test options have already been dealt with. It
553 deals with the core task of building images.
556 args: Command line arguments Namespace object
562 pager = os.getenv('PAGER')
565 fname = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
567 command.Run(pager, fname)
570 # Put these here so that we can import this module without libfdt
571 from binman.image import Image
572 from binman import state
574 if args.cmd in ['ls', 'extract', 'replace']:
576 tout.Init(args.verbosity)
577 tools.PrepareOutputDir(None)
579 ListEntries(args.image, args.paths)
581 if args.cmd == 'extract':
582 ExtractEntries(args.image, args.filename, args.outdir, args.paths,
583 not args.uncompressed)
585 if args.cmd == 'replace':
586 ReplaceEntries(args.image, args.filename, args.indir, args.paths,
587 do_compress=not args.compressed,
588 allow_resize=not args.fix_size, write_map=args.map)
592 tools.FinaliseOutputDir()
595 # Try to figure out which device tree contains our image description
601 raise ValueError('Must provide a board to process (use -b <board>)')
602 board_pathname = os.path.join(args.build_dir, board)
603 dtb_fname = os.path.join(board_pathname, 'u-boot.dtb')
606 args.indir.append(board_pathname)
609 tout.Init(args.verbosity)
610 elf.debug = args.debug
611 cbfs_util.VERBOSE = args.verbosity > 2
612 state.use_fake_dtb = args.fake_dtb
614 tools.SetInputDirs(args.indir)
615 tools.PrepareOutputDir(args.outdir, args.preserve)
616 tools.SetToolPaths(args.toolpath)
617 state.SetEntryArgs(args.entry_arg)
619 images = PrepareImagesAndDtbs(dtb_fname, args.image,
622 for image in images.values():
623 missing |= ProcessImage(image, args.update_fdt, args.map,
624 allow_missing=args.allow_missing)
626 # Write the updated FDTs to our output files
627 for dtb_item in state.GetAllFdts():
628 tools.WriteFile(dtb_item._fname, dtb_item.GetContents())
631 tout.Warning("\nSome images are invalid")
633 tools.FinaliseOutputDir()