From: Artem Bityutskiy Date: Wed, 31 Oct 2012 15:50:53 +0000 (+0200) Subject: Add initial version of bmap-creator and BmapCreator X-Git-Tag: v1.0~168 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=b59b9c85891239f003f76d9b8bd323cf646625d1;p=tools%2Fbmap-tools.git Add initial version of bmap-creator and BmapCreator Change-Id: I3c27b8ba584b6ca4090bb9415a790c24e2b1c3cb Signed-off-by: Artem Bityutskiy --- diff --git a/bmap-creator b/bmap-creator new file mode 100755 index 0000000..4ea6028 --- /dev/null +++ b/bmap-creator @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# +# Copyright (c) 2012 Intel, Inc. +# License: GPLv2 +# Author: Artem Bityutskiy + +# Note! We use the below docstring for the program help text as well. +""" +Generate block map (AKA bmap) for an image. The idea is that while images files +may generally be very large (e.g., 4GiB), they may nevertheless contain only +little real data, e.g., 512MiB. This data are files, directories, file-system +meta-data, partition table, etc. When flashing the image to the target device, +you do not have to copy all the 4GiB of data, you can copy only 512MiB of it, +which is 4 times less, so flashing shoud presumably be 4 times faster. + +The block map file is an XML file which contains a list of blocks which have to +be copied to the target device. The other blocks are not used and there is no +need to copy them. The XML file also contains some additional information like +block size, image size, count of mapped blocks, etc. There are also many +commentaries, so it is human-readable. + +The image file has to be a sparse file. Generally, this often means that when +you generate this image file, you should start with a huge sparse file which +contains a single hole spanning the entire file. Then you should partition it, +write all the data (probably by means of loop-back mounting the image file or +parts of it), etc. The end result should be a sparse file where holes represent +the areas which do not have to be flashed. On the other hand, the mapped file +areas represent the areas which have to be flashed. The block map file lists +these areas. +""" + +VERSION = "0.1.0" + +import argparse +import sys +import logging +from bmaptools import BmapCreator + +def parse_arguments(): + """ A helper function which parses the input arguments. """ + + parser = argparse.ArgumentParser(description = __doc__, + prog = 'bmap-creator') + + # Mandatory command-line argument - image file + text = "the image to generate bmap for" + parser.add_argument("image", help = text) + + # The --output option + text = "the output file name (otherwise stdout is used)" + parser.add_argument("-o", "--output", help = text) + + # The --quiet option + text = "be quiet" + parser.add_argument("-q", "--quiet", action="store_true", help = text) + + # The --no-checksum option + text = "do not generate the checksum for block ranges in the bmap" + parser.add_argument("--no-checksum", action="store_true", help = text) + + # The --version option + parser.add_argument("--version", action="version", \ + version="%(prog)s " + "%s" % VERSION) + + return parser.parse_args() + + +def setup_logger(loglevel): + """ A helper function which sets up and configures the logger. The log + level is initialized to 'loglevel'. Returns the logger object. """ + + # Change log level names to something less nicer than the default + # all-capital 'INFO' etc. + logging.addLevelName(logging.ERROR, "error!") + logging.addLevelName(logging.WARNING, "warning!") + logging.addLevelName(logging.DEBUG, "debug") + logging.addLevelName(logging.INFO, "info") + + log = logging.getLogger('bmap-creator-logger') + log.setLevel(loglevel) + formatter = logging.Formatter("bmap-creator: %(levelname)s: %(message)s") + where = logging.StreamHandler() + where.setFormatter(formatter) + log.addHandler(where) + + return log + +def setup_output_stream(file_path): + """ Create, initialize and return a logger object for the output stream + (where we'll write the bmap). The stream is re-directed to 'file_path' + or to stdout if 'file_path' is None. """ + + output = logging.getLogger('bmap-creator-output') + output.setLevel(logging.INFO) + if file_path: + where = logging.FileHandler(file_path) + else: + where = logging.StreamHandler(sys.stdout) + + output.addHandler(where) + + return output + +def main(): + """ Script entry point. """ + + args = parse_arguments() + + if args.quiet: + log = setup_logger(logging.ERROR) + else: + log = setup_logger(logging.INFO) + + if args.output: + # Make sure the output file is accessible + try: + open(args.output, "w").close() + except IOError as err: + log.error("cannot open the output file '%s': %s" \ + % (args.output, err)) + raise SystemExit(1) + + output = setup_output_stream(args.output) + + try: + creator = BmapCreator.BmapCreator(args.image, output) + creator.generate(not args.no_checksum) + except BmapCreator.Error as err: + log.error(str(err)) + raise SystemExit(1) + + # TODO: complain if no holes + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bmaptools/BmapCreator.py b/bmaptools/BmapCreator.py new file mode 100644 index 0000000..ba64f2c --- /dev/null +++ b/bmaptools/BmapCreator.py @@ -0,0 +1,264 @@ +""" +This module implements the block map (AKA bmap) generating functionality and +provides corresponding API (in a form of the BmapCreator class). + +The idea is that while images files may generally be very large (e.g., 4GiB), +they may nevertheless contain only little real data, e.g., 512MiB. This data +are files, directories, file-system meta-data, partition table, etc. When +flashing the image to the target device, you do not have to copy all the 4GiB +of data, you can copy only 512MiB of it, which is 4 times less, so flashing +shoud presumably be 4 times faster. + +The block map file is an XML file which contains a list of blocks which have to +be copied to the target device. The other blocks are not used and there is no +need to copy them. + +The image file has to be a sparse file. Generally, this often means that when +you generate this image file, you should start with a huge sparse file which +contains a single hole spanning the entire file. Then you should partition it, +write all the data (probably by means of loop-back mounting the image file or +parts of it), etc. The end result should be a sparse file where holes represent +the areas which do not have to be flashed. On the other hand, the mapped file +areas represent the areas which have to be flashed. The block map file lists +these areas. + +At the moment this module uses the FIBMAP ioctl to detect holes. However, it is +possible to speed it up by using presumably faster FIBMAP ioctl (and fall-back +to FIBMAP if the kernel is too old and does not support FIBMAP). +""" + +import os +import hashlib +from fcntl import ioctl +from struct import pack, unpack +from itertools import groupby +from bmaptools import BmapHelpers + +# The bmap format version we generate +bmap_version = "1.2" + +class Error(Exception): + """ A class for exceptions of BmapCreator. We currently support only one + type of exceptions, and we basically throw human-readable problem + description in case of errors. """ + + def __init__(self, strerror, errno = None): + Exception.__init__(self, strerror) + self.strerror = strerror + self.errno = errno + + def __str__(self): + return self.strerror + +class BmapCreator: + """ This class the bmap creation functionality. To generate a bmap for an + image (which is supposedly a sparse file) you should first create an + instance of 'BmapCreator' and provide: + * full path to the image to create bmap for + * a logger object to output the generated bmap to + + Then you should invoke the 'generate()' method of this class. It will + use the FIEMAP ioctl to generate the bmap, and fall-back to the FIBMAP + ioctl if FIEMAP is not supported. """ + + def __init__(self, image_path, output): + """ Initialize a class instance: + * image_path - full path to the image file to generate bmap for + * output - a logger object to write the generated bmap to """ + + self._image_path = image_path + self._output = output + + self.bmap_image_size = None + self.bmap_block_size = None + self.bmap_blocks_cnt = None + self.bmap_mapped_cnt = None + self.bmap_mapped_size = None + self.bmap_mapped_percent = None + + self._f_image = None + + try: + self._f_image = open(image_path, 'rb') + except IOError as err: + raise Error("cannot open image file '%s': %s" \ + % (image_path, err), err.errno) + + self.bmap_image_size = os.fstat(self._f_image.fileno()).st_size + if self.bmap_image_size == 0: + raise Error("cannot generate bmap for zero-sized image file '%s'" \ + % image_path, err.errno) + + # Get the block size of the host file-system for the image file by + # calling the FIGETBSZ ioctl (number 2). + try: + binary_data = ioctl(self._f_image, 2, pack('I', 0)) + self.bmap_block_size = unpack('I', binary_data)[0] + except IOError as err: + raise Error("cannot get block size for '%s': %s" \ + % (image_path, err), err.errno) + + self.bmap_blocks_cnt = self.bmap_image_size + self.bmap_block_size - 1 + self.bmap_blocks_cnt /= self.bmap_block_size + + # Make sure we have enough rights for the FIBMAP ioctl + try: + self._is_mapped(0) + except Error as err: + if err.errno == os.errno.EPERM or err.errno == os.errno.EACCES: + raise Error("you do not have permissions to use the FIBMAP " \ + "ioctl which requires a 'CAP_SYS_RAWIO' " \ + "capability, try to become 'root'", err.errno) + else: + raise + + def _bmap_file_start(self): + """ A helper function which generates the starting contents of the + block map file: the header comment, image size, block size, etc. """ + + xml = "\n\n" + xml += "\n\n" + + xml += "\n" % bmap_version + xml += "\t\n" \ + % BmapHelpers.human_size(self.bmap_image_size) + xml += "\t %u \n\n" % self.bmap_image_size + + xml += "\t\n" + xml += "\t %u \n\n" % self.bmap_block_size + + xml += "\t\n" + xml += "\t %u \n\n" % self.bmap_blocks_cnt + + xml += "\t\n" + xml += "\t" + + self._output.info(xml) + + def _is_mapped(self, block): + """ A helper function which returns True if block number 'block' of the + image file is mapped and False otherwise. + + Implementation details: this function uses the FIBMAP ioctl (number + 1) to get detect whether 'block' is mapped to a disk block. The + ioctl returns zero if 'block' is not mapped and non-zero disk block + number if it is mapped. Unfortunatelly, FIBMAP requires root + rights, unlike FIEMAP. """ + + try: + binary_data = ioctl(self._f_image, 1, pack('I', block)) + result = unpack('I', binary_data)[0] + except IOError as err: + raise Error("the FIBMAP ioctl failed for '%s': %s" \ + % (self._image_path, err), err.errno) + + return result != 0 + + def _get_ranges(self): + """ A helper function which generates ranges of mapped image file + blocks. It uses the FIBMAP ioctl to check which blocks are mapped. + Of course, the image file must have been created as a sparse file + originally, otherwise all blocks will be mapped. And it is also + essential to generate the block map before the file had been copied + anywhere or compressed, because othewise we lose the information + about unmapped blocks. """ + + for key, group in groupby(xrange(self.bmap_blocks_cnt), self._is_mapped): + if key: + # Find the first and the last elements of the group + first = group.next() + last = first + for last in group: + pass + yield first, last + + def _bmap_file_end(self): + """ A helper funstion which generates the final parts of the block map + file: the ending tags and the information about the amount of + mapped blocks. """ + + xml = "\t\n\n" + human_size = BmapHelpers.human_size(self.bmap_mapped_size) + xml += "\t\n" \ + % (human_size, self.bmap_mapped_percent) + xml += "\t %u \n" \ + % self.bmap_mapped_cnt + xml += "" + + self._output.info(xml) + + def _calculate_sha1(self, first, last): + """ A helper function which calculates SHA1 checksum for the range of + blocks of the image file: from block 'first' to block 'last'. """ + + start = first * self.bmap_block_size + end = (last + 1) * self.bmap_block_size + hash_obj = hashlib.sha1() + + chunk_size = 1024*1024 + to_read = end - start + read = 0 + + while read < to_read: + if read + chunk_size > to_read: + chunk_size = to_read - read + chunk = self._f_image.read(chunk_size) + hash_obj.update(chunk) + read += chunk_size + + return hash_obj.hexdigest() + + def generate(self, include_checksums = True): + """ Thenerate bmap for the image file. If 'include_checksums' is True, + also generate SHA1 checksums for block ranges. """ + + self._bmap_file_start() + self._f_image.seek(0) + + # Generate the block map and write it to the XML block map + # file as we go. + self.bmap_mapped_cnt = 0 + for first, last in self._get_ranges(): + self.bmap_mapped_cnt += last - first + 1 + if include_checksums: + sha1 = self._calculate_sha1(first, last) + sha1 = " sha1 =\"%s\"" % sha1 + else: + sha1 = "" + self._output.info("\t\t %s-%s " \ + % (sha1, first, last)) + + self.bmap_mapped_size = self.bmap_mapped_cnt * self.bmap_block_size + self.bmap_mapped_percent = self.bmap_mapped_cnt * 100.0 + self.bmap_mapped_percent /= self.bmap_blocks_cnt + self._bmap_file_end() + + def __del__(self): + """ The class destructor which closes the opened files. """ + + if self._f_image: + self._f_image.close() diff --git a/debian/bmap-tools.install b/debian/bmap-tools.install index 76f1ca4..2e6de41 100644 --- a/debian/bmap-tools.install +++ b/debian/bmap-tools.install @@ -1 +1,2 @@ bmap-flasher usr/bin +bmap-creator usr/bin diff --git a/setup.py b/setup.py index b698fef..386f7b8 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( author = "Artem Bityutskiy", author_email = "artem.bityutskiy@linux.intel.com", version = "0.1.0", - scripts = ['bmap-flasher'], + scripts = ['bmap-flasher', 'bmap-creator'], packages = find_packages(), license='GPLv2', long_description="Tools to generate block map (AKA bmap) and flash " \