From b3392b69bd95518b919454da8258863cc6dfed9e Mon Sep 17 00:00:00 2001 From: Artem Bityutskiy Date: Thu, 8 Nov 2012 13:33:41 +0200 Subject: [PATCH] Implement sub-commands support Remove separate 'bmap-creator' and 'bmap-flasher' tools and instead, implement a single 'bmap' tool with 'create' and 'flash' subcommands. Change-Id: I6c0d0b2aee94e7996db5dcac6a5513da7f47e067 Signed-off-by: Artem Bityutskiy --- bmap | 240 ++++++++++++++++++++++++++++++++++++++++++++++ bmap-creator | 139 --------------------------- bmap-flasher | 150 ----------------------------- debian/bmap-tools.install | 3 +- setup.py | 2 +- 5 files changed, 242 insertions(+), 292 deletions(-) create mode 100755 bmap delete mode 100755 bmap-creator delete mode 100755 bmap-flasher diff --git a/bmap b/bmap new file mode 100755 index 0000000..9f74650 --- /dev/null +++ b/bmap @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# +# Copyright (c) 2012 Intel, Inc. +# License: GPLv2 +# Author: Artem Bityutskiy + +# Note! We use the below docstring for the program help text as well. +""" A tool to generate block map (AKA bmap) for image files and to copy or +flash image files using bmap. + +The bmap is an XML file which which contains list of mapped blocks in the +image. Mapped blocks are image file blocks which have corresponding disk +sectors associated with them, as opposed to holes which are image file blocks +not associated with any disk sectors. In other words, the image is considered +to be a sparse file and bmap basically contains a list of mapped blocks of this +sparse file. The bmap also contains some additional useful information like +block size (usually 4KiB), image size, mapped blocks count, etc. + +The bmap is used for flashing or copying the image to a block device or a +different file. The idea is that we may flash or copy quickly with the bmap +because we copy only mapped blocks and ignore the holes, because they are +useless. And usually images contain a lot of unused blocks (of course, we +assume that the image was a sparse file at the moment of creation, and the bmap +was generated for this sparse file). + +For example, you may have a 4GiB image file, which contains only 100MiB of user +data and you need to flashi it to a slow USB stick. With the bmap file you end +up writing only a little bit more than 100MiB of data from the image file to +the USB stick. This is a lot faster than writing all 4GiB of data. We say that +it is a bit more than 100MiB because there are also file-system meta-data, +partition table, etc. """ + +VERSION = "0.1.1" + +import argparse +import sys +import time +import logging +from bmaptools import BmapCreate, BmapFlash, BmapHelpers + +def flash_command(args, log): + """ Write raw image file to the target block device using the block map + file (AKA bmap). The bmap contains list of blocks which have to be read + from the image file and then written to the block device. The rest of the + blocks are not required to be copied. And usually image files have a lot of + useless blocks (i.e., the blocks which are not used in the internal + file-system of the image), so flashing with bmap is usually much faster + than copying entire image to the block device. """ + + try: + flasher = BmapFlash.BmapFlash(args.image, args.bdev, args.bmap) + except BmapFlash.Error as err: + log.error(str(err)) + raise SystemExit(1) + + if not flasher.target_is_block_device: + log.warning("'%s' is not a block device!" % args.bdev) + + start_time = time.time() + if not args.bmap: + log.info("no block map given (see the --bmap option)") + log.info("falling-back to writing entire image to '%s'" % args.bdev) + else: + log.info("block map format version %s" % flasher.bmap_version) + log.info("%d blocks of size %d (%s), mapped %d blocks (%s or %.1f%%)" \ + % (flasher.bmap_blocks_cnt, flasher.bmap_block_size, + flasher.bmap_image_size_human, flasher.bmap_mapped_cnt, + flasher.bmap_mapped_size_human, + flasher.bmap_mapped_percent)) + log.info("writing the image to '%s' using bmap file '%s'" \ + % (args.bdev, args.bmap)) + + try: + try: + flasher.write(False, not args.no_verify) + except BmapFlash.Error as err: + log.error(str(err)) + raise SystemExit(1) + + # Synchronize the block device + log.info("synchronizing block device '%s'" % args.bdev) + try: + flasher.sync() + except BmapFlash.Error as err: + log.error(str(err)) + raise SystemExit(1) + except KeyboardInterrupt: + log.error("the program is interrupted") + log.warning("do not panic if the program may not finish immediately, " \ + "just wait") + log.warning("reason: this is the Linux kernel behavior - it " \ + "synchronizes the block device") + raise SystemExit(1) + + flashing_time = time.time() - start_time + flashing_speed = flasher.bmap_image_size / flashing_time + log.info("flashing time: %s, flashing speed %s/sec" \ + % (BmapHelpers.human_time(flashing_time), \ + BmapHelpers.human_size(flashing_speed))) + +def create_command(args, log): + """ Generate block map (AKA bmap) for an image. The idea is that while + images files may generally be very large (e.g., 4GiB), they may + nevertheless contain only little real data, e.g., 512MiB. This data are + files, directories, file-system meta-data, partition table, etc. When + flashing the image to the target device, you do not have to copy all the + 4GiB of data, you can copy only 512MiB of it, which is 4 times less, so + flashing shoud presumably be 4 times faster. + + The block map file is an XML file which contains a list of blocks which + have to be copied to the target device. The other blocks are not used and + there is no need to copy them. The XML file also contains some additional + information like block size, image size, count of mapped blocks, etc. There + are also many commentaries, so it is human-readable. + + The image file has to be a sparse file. Generally, this often means that + when you generate this image file, you should start with a huge sparse file + which contains a single hole spanning the entire file. Then you should + partition it, write all the data (probably by means of loop-back mounting + the image file or parts of it), etc. The end result should be a sparse + file where holes represent the areas which do not have to be flashed. On + the other hand, the mapped file areas represent the areas which have to be + flashed. The block map file lists these areas. """ + + # Create and setup the output stream + output = logging.getLogger('bmap-create-output') + output.setLevel(logging.INFO) + if args.output: + where = logging.FileHandler(args.output) + else: + where = logging.StreamHandler(sys.stdout) + output.addHandler(where) + + try: + creator = BmapCreate.BmapCreate(args.image, output) + creator.generate(not args.no_checksum) + except BmapCreate.Error as err: + log.error(str(err)) + raise SystemExit(1) + + if creator.bmap_mapped_cnt == creator.bmap_blocks_cnt: + log.warning("all %s are mapped, no holes in '%s'" \ + % (creator.bmap_image_size_human, args.image)) + log.warning("was the image handled incorrectly and holes " \ + "were expanded?") + +def parse_arguments(): + """ A helper function which parses the input arguments. """ + + parser = argparse.ArgumentParser(description = __doc__, prog = 'bmap') + + # The --version option + parser.add_argument("--version", action = "version", \ + version = "%(prog)s " + "%s" % VERSION) + + # The --quiet option + text = "be quiet" + parser.add_argument("-q", "--quiet", action = "store_true", help = text) + + subparsers = parser.add_subparsers(title = "subcommands") + + # + # Create the parser for the "create" command + # + text = "generate bmap for an image file (which should be a sparse file)" + parser_create = subparsers.add_parser("create", help = text) + parser_create.set_defaults(func=create_command) + + # Mandatory command-line argument - image file + text = "the image to generate bmap for" + parser_create.add_argument("image", help = text) + + # The --output option + text = "the output file name (otherwise stdout is used)" + parser_create.add_argument("-o", "--output", help = text) + + # The --no-checksum option + text = "do not generate the checksum for block ranges in the bmap" + parser_create.add_argument("--no-checksum", action="store_true", help = text) + + # + # Create the parser for the "flash" command + # + text = "write an image to a block device using bmap" + parser_flash = subparsers.add_parser("flash", help = text) + parser_flash.set_defaults(func=flash_command) + + # The first positional argument - image file + text = "the image file to flash. Supported formats: uncompressed, " + \ + ", ".join(BmapFlash.supported_image_formats) + parser_flash.add_argument("image", help = text) + + # The second positional argument - block device node + text = "the block device node to flash the image to" + parser_flash.add_argument("bdev", help = text) + + # The --bmap option + text = "the block map file for the image" + parser_flash.add_argument("--bmap", help = text) + + # The --no-verify option + text = "do not verify the data checksum while writing" + parser_flash.add_argument("--no-verify", action="store_true", help = text) + + return parser.parse_args() + +def setup_logger(loglevel): + """ A helper function which sets up and configures the logger. The log + level is initialized to 'loglevel'. Returns the logger object. """ + + # Change log level names to something less nicer than the default + # all-capital 'INFO' etc. + logging.addLevelName(logging.ERROR, "error!") + logging.addLevelName(logging.WARNING, "warning!") + logging.addLevelName(logging.DEBUG, "debug") + logging.addLevelName(logging.INFO, "info") + + log = logging.getLogger('bmap-logger') + log.setLevel(loglevel) + formatter = logging.Formatter("bmap: %(levelname)s: %(message)s") + where = logging.StreamHandler(sys.stderr) + where.setFormatter(formatter) + log.addHandler(where) + + return log + +def main(): + """ Script entry point. """ + + args = parse_arguments() + + if args.quiet: + loglevel = logging.ERROR + else: + loglevel = logging.INFO + + args.func(args, setup_logger(loglevel)) + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bmap-creator b/bmap-creator deleted file mode 100755 index 88829f7..0000000 --- a/bmap-creator +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2012 Intel, Inc. -# License: GPLv2 -# Author: Artem Bityutskiy - -# Note! We use the below docstring for the program help text as well. -""" -Generate block map (AKA bmap) for an image. The idea is that while images files -may generally be very large (e.g., 4GiB), they may nevertheless contain only -little real data, e.g., 512MiB. This data are files, directories, file-system -meta-data, partition table, etc. When flashing the image to the target device, -you do not have to copy all the 4GiB of data, you can copy only 512MiB of it, -which is 4 times less, so flashing shoud presumably be 4 times faster. - -The block map file is an XML file which contains a list of blocks which have to -be copied to the target device. The other blocks are not used and there is no -need to copy them. The XML file also contains some additional information like -block size, image size, count of mapped blocks, etc. There are also many -commentaries, so it is human-readable. - -The image file has to be a sparse file. Generally, this often means that when -you generate this image file, you should start with a huge sparse file which -contains a single hole spanning the entire file. Then you should partition it, -write all the data (probably by means of loop-back mounting the image file or -parts of it), etc. The end result should be a sparse file where holes represent -the areas which do not have to be flashed. On the other hand, the mapped file -areas represent the areas which have to be flashed. The block map file lists -these areas. -""" - -VERSION = "0.1.1" - -import argparse -import sys -import logging -from bmaptools import BmapCreate - -def parse_arguments(): - """ A helper function which parses the input arguments. """ - - parser = argparse.ArgumentParser(description = __doc__, - prog = 'bmap-creator') - - # Mandatory command-line argument - image file - text = "the image to generate bmap for" - parser.add_argument("image", help = text) - - # The --output option - text = "the output file name (otherwise stdout is used)" - parser.add_argument("-o", "--output", help = text) - - # The --quiet option - text = "be quiet" - parser.add_argument("-q", "--quiet", action="store_true", help = text) - - # The --no-checksum option - text = "do not generate the checksum for block ranges in the bmap" - parser.add_argument("--no-checksum", action="store_true", help = text) - - # The --version option - parser.add_argument("--version", action="version", \ - version="%(prog)s " + "%s" % VERSION) - - return parser.parse_args() - - -def setup_logger(loglevel): - """ A helper function which sets up and configures the logger. The log - level is initialized to 'loglevel'. Returns the logger object. """ - - # Change log level names to something less nicer than the default - # all-capital 'INFO' etc. - logging.addLevelName(logging.ERROR, "error!") - logging.addLevelName(logging.WARNING, "warning!") - logging.addLevelName(logging.DEBUG, "debug") - logging.addLevelName(logging.INFO, "info") - - log = logging.getLogger('bmap-creator-logger') - log.setLevel(loglevel) - formatter = logging.Formatter("bmap-creator: %(levelname)s: %(message)s") - where = logging.StreamHandler() - where.setFormatter(formatter) - log.addHandler(where) - - return log - -def setup_output_stream(file_path): - """ Create, initialize and return a logger object for the output stream - (where we'll write the bmap). The stream is re-directed to 'file_path' - or to stdout if 'file_path' is None. """ - - output = logging.getLogger('bmap-creator-output') - output.setLevel(logging.INFO) - if file_path: - where = logging.FileHandler(file_path) - else: - where = logging.StreamHandler(sys.stdout) - - output.addHandler(where) - - return output - -def main(): - """ Script entry point. """ - - args = parse_arguments() - - if args.quiet: - log = setup_logger(logging.ERROR) - else: - log = setup_logger(logging.INFO) - - if args.output: - # Make sure the output file is accessible - try: - open(args.output, "w").close() - except IOError as err: - log.error("cannot open the output file '%s': %s" \ - % (args.output, err)) - raise SystemExit(1) - - output = setup_output_stream(args.output) - - try: - creator = BmapCreate.BmapCreate(args.image, output) - creator.generate(not args.no_checksum) - except BmapCreate.Error as err: - log.error(str(err)) - raise SystemExit(1) - - if creator.bmap_mapped_cnt == creator.bmap_blocks_cnt: - log.warning("all %s are mapped, no holes in '%s'" \ - % (creator.bmap_image_size_human, args.image)) - log.warning("was the image handled incorrectly and holes " \ - "were expanded?") - -if __name__ == "__main__": - sys.exit(main()) diff --git a/bmap-flasher b/bmap-flasher deleted file mode 100755 index 504eb1c..0000000 --- a/bmap-flasher +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (c) 2012 Intel, Inc. -# License: GPLv2 -# Author: Artem Bityutskiy - -# Note! We use the below docstring for the program help text as well. -""" -Write raw image file to the target block device using the block map file (AKA -bmap). The bmap contains list of blocks which have to be read from the image -file and then written to the block device. The rest of the blocks are not -required to be copied. And usually image files have a lot of useless blocks -(i.e., the blocks which are not used in the internal file-system of the image), -so flashing with bmap is usually much faster than copying entire image to the -block device. - -For example, you may have a 4GiB image file, which contains only 100MiB of user -data. In this case, with the bmap file you will write only a little bit more -than 100MiB of data from the image file to the block device. This is a lot -faster than writing the entire 4GiB image. We say that it is a bit more than -100MiB because there are also file-system meta-data, partition table, etc. The -bmap fail is quite human-readable and contains a lot of commentaries. But -essentially, it is an XML document which contains list of blocks in the image -file which have to be copied to the block device. -""" - -VERSION = "0.1.1" - -import argparse -import sys -import time -import logging -from bmaptools import BmapFlash, BmapHelpers - -def parse_arguments(): - """ A helper function which parses the input arguments. """ - - parser = argparse.ArgumentParser(description = __doc__, - prog = 'bmap-flasher') - - # The first positional argument - image file - text = "the image file to flash. Supported formats: uncompressed, " + \ - ", ".join(BmapFlash.supported_image_formats) - parser.add_argument("image", help = text) - - # The second positional argument - block device node - text = "the block device node to flash the image to" - parser.add_argument("bdev", help = text) - - # The --bmap option - text = "the block map file for the image" - parser.add_argument("--bmap", help = text) - - # The --no-verify option - text = "do not verify the data checksum while writing" - parser.add_argument("--no-verify", action="store_true", help = text) - - # The --quiet option - text = "be quiet" - parser.add_argument("-q", "--quiet", action="store_true", help = text) - - # The --version option - parser.add_argument("--version", action="version", \ - version="%(prog)s " + "%s" % VERSION) - - return parser.parse_args() - -def setup_logger(loglevel): - """ A helper function which sets up and configures the logger. The log - level is initialized to 'loglevel'. Returns the logger object. """ - - # Change log level names to something less nicer than the default - # all-capital 'INFO' etc. - logging.addLevelName(logging.ERROR, "error!") - logging.addLevelName(logging.WARNING, "warning!") - logging.addLevelName(logging.DEBUG, "debug") - logging.addLevelName(logging.INFO, "info") - - log = logging.getLogger('bmap-flasher-logger') - log.setLevel(loglevel) - formatter = logging.Formatter("bmap-flasher: %(levelname)s: %(message)s") - where = logging.StreamHandler() - where.setFormatter(formatter) - log.addHandler(where) - - return log - -def main(): - """ Script entry point. """ - - args = parse_arguments() - - if args.quiet: - log = setup_logger(logging.ERROR) - else: - log = setup_logger(logging.INFO) - - try: - flasher = BmapFlash.BmapFlash(args.image, args.bdev, args.bmap) - except BmapFlash.Error as err: - log.error(str(err)) - raise SystemExit(1) - - if not flasher.target_is_block_device: - log.warning("'%s' is not a block device!" % args.bdev) - - start_time = time.time() - if not args.bmap: - log.info("no block map given (see the --bmap option)") - log.info("falling-back to writing entire image to '%s'" % args.bdev) - else: - log.info("block map format version %s" % flasher.bmap_version) - log.info("%d blocks of size %d (%s), mapped %d blocks (%s or %.1f%%)" \ - % (flasher.bmap_blocks_cnt, flasher.bmap_block_size, - flasher.bmap_image_size_human, flasher.bmap_mapped_cnt, - flasher.bmap_mapped_size_human, - flasher.bmap_mapped_percent)) - log.info("writing the image to '%s' using bmap file '%s'" \ - % (args.bdev, args.bmap)) - - try: - try: - flasher.write(False, not args.no_verify) - except BmapFlash.Error as err: - log.error(str(err)) - raise SystemExit(1) - - # Synchronize the block device - log.info("synchronizing block device '%s'" % args.bdev) - try: - flasher.sync() - except BmapFlash.Error as err: - log.error(str(err)) - raise SystemExit(1) - except KeyboardInterrupt: - log.error("the program is interrupted") - log.warning("do not panic if the program may not finish immediately, " \ - "just wait") - log.warning("reason: this is the Linux kernel behavior - it " \ - "synchronizes the block device") - raise SystemExit(1) - - flashing_time = time.time() - start_time - flashing_speed = flasher.bmap_image_size / flashing_time - log.info("flashing time: %s, flashing speed %s/sec" \ - % (BmapHelpers.human_time(flashing_time), \ - BmapHelpers.human_size(flashing_speed))) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/debian/bmap-tools.install b/debian/bmap-tools.install index 2e6de41..3ebbd3a 100644 --- a/debian/bmap-tools.install +++ b/debian/bmap-tools.install @@ -1,2 +1 @@ -bmap-flasher usr/bin -bmap-creator usr/bin +bmap usr/bin diff --git a/setup.py b/setup.py index 386f7b8..57100eb 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ setup( author = "Artem Bityutskiy", author_email = "artem.bityutskiy@linux.intel.com", version = "0.1.0", - scripts = ['bmap-flasher', 'bmap-creator'], + scripts = ['bmap'], packages = find_packages(), license='GPLv2', long_description="Tools to generate block map (AKA bmap) and flash " \ -- 2.7.4