From 6ef271e006bf4b1dc18f0e4b0c147590e1946225 Mon Sep 17 00:00:00 2001 From: Artem Bityutskiy Date: Wed, 3 Jul 2013 17:00:41 +0300 Subject: [PATCH] bmaptool: use PEP8 commenting style No functional changes, just amend multiline comments to match PEP8's recommendation. Change-Id: I91aa25026d73d2b6fad4230705d2bc1f14d4fb91 Signed-off-by: Artem Bityutskiy --- bmaptool | 66 ++++++++++++++---------- bmaptools/BmapCopy.py | 131 +++++++++++++++++++++++++++++------------------ bmaptools/BmapCreate.py | 62 +++++++++++++--------- bmaptools/BmapHelpers.py | 10 ++-- bmaptools/Fiemap.py | 80 ++++++++++++++++++----------- bmaptools/TransRead.py | 123 ++++++++++++++++++++++++++------------------ 6 files changed, 290 insertions(+), 182 deletions(-) diff --git a/bmaptool b/bmaptool index 24356b9..a58f797 100755 --- a/bmaptool +++ b/bmaptool @@ -4,9 +4,9 @@ # License: GPLv2 # Author: Artem Bityutskiy -""" This is a tool to generate block map files (bmap) and to copy files using -bmap. Generally speaking, these tools are about writing large image files -quickly. +""" +This is a tool to generate block map files (bmap) and to copy files using bmap. +Generally speaking, these tools are about writing large image files quickly. The bmap file is an XML file which contains a list of mapped blocks of the image. Mapped blocks are the blocks which have disk sectors associated with @@ -30,7 +30,8 @@ copying only a little bit more than 100MiB of data from the image to the USB stick (namely, you write only mapped blocks). This is a lot faster than copying all 4GiB of data. We say that it is a bit more than 100MiB because things like file-system meta-data (inode tables, superblocks, etc), partition table, etc -also contribute to the mapped blocks and are also copied. """ +also contribute to the mapped blocks and are also copied. +""" # Disable the following pylint recommendations: # * Too few public methods (R0903) @@ -49,13 +50,17 @@ import traceback from bmaptools import BmapCreate, BmapCopy, BmapHelpers, TransRead def copy_command_open_blkdev(path, log): - """ Open a block device specified by 'path' in exclusive mode. Returns - opened file object. """ + """ + Open a block device specified by 'path' in exclusive mode. Returns opened + file object. + """ class NamedFile: - """ This simple class allows us to override the 'name' attribute of a - file object. The problem is that 'os.fdopen()' sets the name to - "", which is not very user-friendly. """ + """ + This simple class allows us to override the 'name' attribute of a file + object. The problem is that 'os.fdopen()' sets the name to "", + which is not very user-friendly. + """ def __init__(self, file_obj, name): self._file_obj = file_obj @@ -82,14 +87,16 @@ def copy_command_open_blkdev(path, log): return NamedFile(file_obj, path) def find_and_open_bmap(image_path): - """ When the user does not specify the bmap file, we try to find it at the - same place where the image file is located. We search for a file with the - same path and basename, but with a ".bmap" extension. Since the image may + """ + When the user does not specify the bmap file, we try to find it at the same + place where the image file is located. We search for a file with the same + path and basename, but with a ".bmap" extension. Since the image may contain more than one extension (e.g., image.raw.bz2), try to remove them one-by-one. This function returns a file-like object for the bmap file if it has been - found, and 'None' otherwise. """ + found, and 'None' otherwise. + """ bmap_path = None @@ -110,7 +117,8 @@ def find_and_open_bmap(image_path): return None def copy_command_open_all(args, log): - """ Open the image/bmap/destination files for the "copy" command. Returns a + """ + Open the image/bmap/destination files for the "copy" command. Returns a tuple of 5 elements: 1 file-like object for the image 2 file object for the destination file @@ -167,7 +175,7 @@ def copy_command_open_all(args, log): def copy_command(args, log): - """ Copy an image to a block device or a regular file using bmap. """ + """Copy an image to a block device or a regular file using bmap.""" image_obj, dest_obj, bmap_obj, bmap_path, image_size, dest_is_blkdev = \ copy_command_open_all(args, log) @@ -236,13 +244,14 @@ def copy_command(args, log): image_obj.close() def create_command(args, log): - """ Generate block map (AKA bmap) for an image. The idea is that while - images files may generally be very large (e.g., 4GiB), they may - nevertheless contain only little real data, e.g., 512MiB. This data are - files, directories, file-system meta-data, partition table, etc. When - copying the image to the target device, you do not have to copy all the - 4GiB of data, you can copy only 512MiB of it, which is 4 times less, so - copying should presumably be 4 times faster. + """ + Generate block map (AKA bmap) for an image. The idea is that while images + files may generally be very large (e.g., 4GiB), they may nevertheless + contain only little real data, e.g., 512MiB. This data are files, + directories, file-system meta-data, partition table, etc. When copying the + image to the target device, you do not have to copy all the 4GiB of data, + you can copy only 512MiB of it, which is 4 times less, so copying should + presumably be 4 times faster. The block map file is an XML file which contains a list of blocks which have to be copied to the target device. The other blocks are not used and @@ -257,7 +266,8 @@ def create_command(args, log): or parts of it), etc. The end result should be a sparse file where mapped areas represent useful parts of the image and holes represent useless parts of the image, which do not have to be copied when copying the image to the - target device. """ + target device. + """ # Create and setup the output stream if args.output: @@ -293,7 +303,7 @@ def create_command(args, log): "were expanded?") def parse_arguments(): - """ A helper function which parses the input arguments. """ + """A helper function which parses the input arguments.""" text = "Create block map (bmap) and copy files using bmap. The " \ "documentation can be found here: " \ @@ -361,8 +371,10 @@ def parse_arguments(): return parser.parse_args() def setup_logger(loglevel): - """ A helper function which sets up and configures the logger. The log - level is initialized to 'loglevel'. Returns the logger object. """ + """ + A helper function which sets up and configures the logger. The log level is + initialized to 'loglevel'. Returns the logger object. + """ # Esc-sequences for coloured output esc_red = '\033[91m' # pylint: disable=W1401 @@ -386,7 +398,7 @@ def setup_logger(loglevel): return log def main(): - """ Script entry point. """ + """Script entry point.""" args = parse_arguments() diff --git a/bmaptools/BmapCopy.py b/bmaptools/BmapCopy.py index f7a4ace..ac5dfc8 100644 --- a/bmaptools/BmapCopy.py +++ b/bmaptools/BmapCopy.py @@ -1,5 +1,6 @@ -""" This module implements copying of images with bmap and provides the -following API. +""" +This module implements copying of images with bmap and provides the following +API. 1. BmapCopy class - implements copying to any kind of file, be that a block device or a regular file. 2. BmapBdevCopy class - based on BmapCopy and specializes on copying to block @@ -28,7 +29,8 @@ copying only a little bit more than 100MiB of data from the image to the USB stick (namely, you copy only mapped blocks). This is a lot faster than copying all 4GiB of data. We say that it is a bit more than 100MiB because things like file-system meta-data (inode tables, superblocks, etc), partition table, etc -also contribute to the mapped blocks and are also copied. """ +also contribute to the mapped blocks and are also copied. +""" # Disable the following pylint recommendations: # * Too many instance attributes (R0902) @@ -49,13 +51,16 @@ from bmaptools.BmapHelpers import human_size SUPPORTED_BMAP_VERSION = 1 class Error(Exception): - """ A class for exceptions generated by the 'BmapCopy' module. We currently + """ + A class for exceptions generated by the 'BmapCopy' module. We currently support only one type of exceptions, and we basically throw human-readable - problem description in case of errors. """ + problem description in case of errors. + """ pass class BmapCopy: - """ This class implements the bmap-based copying functionality. To copy an + """ + This class implements the bmap-based copying functionality. To copy an image with bmap you should create an instance of this class, which requires the following: @@ -98,18 +103,21 @@ class BmapCopy: You can copy only once with an instance of this class. This means that in order to copy the image for the second time, you have to create a new class - instance. """ + instance. + """ def set_progress_indicator(self, file_obj, format_string): - """ Setup the progress indicator which shows how much data has been - copied in percent. + """ + Setup the progress indicator which shows how much data has been copied + in percent. The 'file_obj' argument is the console file object where the progress has to be printed to. Pass 'None' to disable the progress indicator. The 'format_string' argument is the format string for the progress indicator. It has to contain a single '%d' placeholder which will be - substitutes with copied data in percent. """ + substitutes with copied data in percent. + """ self._progress_file = file_obj if format_string: @@ -118,8 +126,9 @@ class BmapCopy: self._progress_format = "Copied %d%%" def _set_image_size(self, image_size): - """ Set image size and initialize various other geometry-related - attributes. """ + """ + Set image size and initialize various other geometry-related attributes. + """ if self.image_size is not None and self.image_size != image_size: raise Error("cannot set image size to %d bytes, it is known to " \ @@ -137,8 +146,9 @@ class BmapCopy: self.mapped_size_human = self.image_size_human def _verify_bmap_checksum(self): - """ This is a helper function which verifies SHA1 checksum of the bmap - file. """ + """ + This is a helper function which verifies SHA1 checksum of the bmap file. + """ import mmap @@ -164,8 +174,9 @@ class BmapCopy: (self._bmap_path, calculated_sha1, correct_sha1)) def _parse_bmap(self): - """ Parse the bmap file and initialize corresponding class instance - attributs. """ + """ + Parse the bmap file and initialize corresponding class instance attributs. + """ try: self._xml = ElementTree.parse(self._f_bmap) @@ -206,7 +217,8 @@ class BmapCopy: def __init__(self, image, dest, bmap = None, image_size = None, logger = None): - """ The class constructor. The parameters are: + """ + The class constructor. The parameters are: image - file-like object of the image which should be copied, should only support 'read()' and 'seek()' methods, and only seeking forward has to be supported. @@ -215,7 +227,7 @@ class BmapCopy: bmap - file object of the bmap file to use for copying. image_size - size of the image in bytes. logger - the logger object to use for printing messages. - """ + """ self._logger = logger if self._logger is None: @@ -284,9 +296,11 @@ class BmapCopy: self._batch_blocks = self._batch_bytes / self.block_size def _update_progress(self, blocks_written): - """ Print the progress indicator if the mapped area size is known and - if the indicator has been enabled by assigning a console file object to - the 'progress_file' attribute. """ + """ + Print the progress indicator if the mapped area size is known and if + the indicator has been enabled by assigning a console file object to + the 'progress_file' attribute. + """ if not self._progress_file: return @@ -322,9 +336,10 @@ class BmapCopy: self._progress_file.flush() def _get_block_ranges(self): - """ This is a helper generator that parses the bmap XML file and for - each block range in the XML file it yields ('first', 'last', 'sha1') - tuples, where: + """ + This is a helper generator that parses the bmap XML file and for each + block range in the XML file it yields ('first', 'last', 'sha1') tuples, + where: * 'first' is the first block of the range; * 'last' is the last block of the range; * 'sha1' is the SHA1 checksum of the range ('None' is used if it is @@ -332,7 +347,8 @@ class BmapCopy: If there is no bmap file, the generator just yields a single range for entire image file. If the image size is unknown, the generator - infinitely yields continuous ranges of size '_batch_blocks'. """ + infinitely yields continuous ranges of size '_batch_blocks'. + """ if not self._f_bmap: # We do not have the bmap, yield a tuple with all blocks @@ -374,8 +390,9 @@ class BmapCopy: yield (first, last, sha1) def _get_batches(self, first, last): - """ This is a helper generator which splits block ranges from the bmap - file to smaller batches. Indeed, we cannot read and write entire block + """ + This is a helper generator which splits block ranges from the bmap file + to smaller batches. Indeed, we cannot read and write entire block ranges from the image file, because a range can be very large. So we perform the I/O in batches. Batch size is defined by the '_batch_blocks' attribute. Thus, for each (first, last) block range, @@ -383,7 +400,8 @@ class BmapCopy: * 'start' is the starting batch block number; * 'last' is the ending batch block number; * 'length' is the batch length in blocks (same as - 'end' - 'start' + 1). """ + 'end' - 'start' + 1). + """ batch_blocks = self._batch_blocks @@ -396,11 +414,13 @@ class BmapCopy: yield (first, first + batch_blocks - 1, batch_blocks) def _get_data(self, verify): - """ This is generator which reads the image file in '_batch_blocks' - chunks and yields ('type', 'start', 'end', 'buf) tuples, where: + """ + This is generator which reads the image file in '_batch_blocks' chunks + and yields ('type', 'start', 'end', 'buf) tuples, where: * 'start' is the starting block number of the batch; * 'end' is the last block of the batch; - * 'buf' a buffer containing the batch data. """ + * 'buf' a buffer containing the batch data. + """ try: for (first, last, sha1) in self._get_block_ranges(): @@ -445,10 +465,12 @@ class BmapCopy: self._batch_queue.put(None) def copy(self, sync = True, verify = True): - """ Copy the image to the destination file using bmap. The 'sync' - argument defines whether the destination file has to be synchronized - upon return. The 'verify' argument defines whether the SHA1 checksum - has to be verified while copying. """ + """ + Copy the image to the destination file using bmap. The 'sync' argument + defines whether the destination file has to be synchronized upon + return. The 'verify' argument defines whether the SHA1 checksum has to + be verified while copying. + """ # Create the queue for block batches and start the reader thread, which # will read the image in batches and put the results to '_batch_queue'. @@ -531,8 +553,10 @@ class BmapCopy: self.sync() def sync(self): - """ Synchronize the destination file to make sure all the data are - actually written to the disk. """ + """ + Synchronize the destination file to make sure all the data are actually + written to the disk. + """ if self._dest_supports_fsync: try: @@ -543,13 +567,16 @@ class BmapCopy: class BmapBdevCopy(BmapCopy): - """ This class is a specialized version of 'BmapCopy' which copies the - image to a block device. Unlike the base 'BmapCopy' class, this class does - various optimizations specific to block devices, e.g., switching to the - 'noop' I/O scheduler. """ + """ + This class is a specialized version of 'BmapCopy' which copies the image to + a block device. Unlike the base 'BmapCopy' class, this class does various + optimizations specific to block devices, e.g., switching to the 'noop' I/O + scheduler. + """ def _tune_block_device(self): - """" Tune the block device for better performance: + """ + Tune the block device for better performance: 1. Switch to the 'noop' I/O scheduler if it is available - sequential write to the block device becomes a lot faster comparing to CFQ. 2. Limit the write buffering - we do not need the kernel to buffer a @@ -593,8 +620,10 @@ class BmapBdevCopy(BmapCopy): "I/O scheduler: %s)" % err) def _restore_bdev_settings(self): - """ Restore old block device settings which we changed in - '_tune_block_device()'. """ + """ + Restore old block device settings which we changed in + '_tune_block_device()'. + """ if self._old_scheduler_value is not None: try: @@ -613,16 +642,16 @@ class BmapBdevCopy(BmapCopy): % (self._old_max_ratio_value, err)) def copy(self, sync = True, verify = True): - """ The same as in the base class but tunes the block device for better - performance before starting writing. Additionally, it forces block - device synchronization from time to time in order to make sure we do + """ + The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do not get stuck in 'fsync()' for too long time. The problem is that the kernel synchronizes block devices when the file is closed. And the result is that if the user interrupts us while we are copying the data, the program will be blocked in 'close()' waiting for the block device synchronization, which may last minutes for slow USB stick. This is very bad user experience, and we work around this effect by - synchronizing from time to time. """ + synchronizing from time to time. + """ self._tune_block_device() @@ -635,8 +664,10 @@ class BmapBdevCopy(BmapCopy): def __init__(self, image, dest, bmap = None, image_size = None, logger = None): - """ The same as the constructor of the 'BmapCopy' base class, but adds - useful guard-checks specific to block devices. """ + """ + The same as the constructor of the 'BmapCopy' base class, but adds + useful guard-checks specific to block devices. + """ # Call the base class constructor first BmapCopy.__init__(self, image, dest, bmap, image_size, logger=logger) diff --git a/bmaptools/BmapCreate.py b/bmaptools/BmapCreate.py index 7362816..d60e639 100644 --- a/bmaptools/BmapCreate.py +++ b/bmaptools/BmapCreate.py @@ -1,5 +1,6 @@ -""" This module implements the block map (bmap) creation functionality and -provides the corresponding API in form of the 'BmapCreate' class. +""" +This module implements the block map (bmap) creation functionality and provides +the corresponding API in form of the 'BmapCreate' class. The idea is that while images files may generally be very large (e.g., 4GiB), they may nevertheless contain only little real data, e.g., 512MiB. This data @@ -22,7 +23,8 @@ etc. The end result should be a sparse file where mapped areas represent useful parts of the image and holes represent useless parts of the image, which do not have to be copied when copying the image to the target device. -This module uses the FIBMAP ioctl to detect holes. """ +This module uses the FIBMAP ioctl to detect holes. +""" # Disable the following pylint recommendations: # * Too many instance attributes - R0902 @@ -73,24 +75,28 @@ _BMAP_START_TEMPLATE = \ """ class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ + """ + A class for exceptions generated by this module. We currently support only + one type of exceptions, and we basically throw human-readable problem + description in case of errors. + """ pass class BmapCreate: - """ This class implements the bmap creation functionality. To generate a - bmap for an image (which is supposedly a sparse file), you should first - create an instance of 'BmapCreate' and provide: + """ + This class implements the bmap creation functionality. To generate a bmap + for an image (which is supposedly a sparse file), you should first create + an instance of 'BmapCreate' and provide: * full path or a file-like object of the image to create bmap for * full path or a file object to use for writing the results to Then you should invoke the 'generate()' method of this class. It will use - the FIEMAP ioctl to generate the bmap. """ + the FIEMAP ioctl to generate the bmap. + """ def _open_image_file(self): - """ Open the image file. """ + """Open the image file.""" try: self._f_image = open(self._image_path, 'rb') @@ -101,7 +107,7 @@ class BmapCreate: self._f_image_needs_close = True def _open_bmap_file(self): - """ Open the bmap file. """ + """Open the bmap file.""" try: self._f_bmap = open(self._bmap_path, 'w+') @@ -112,11 +118,13 @@ class BmapCreate: self._f_bmap_needs_close = True def __init__(self, image, bmap): - """ Initialize a class instance: + """ + Initialize a class instance: * image - full path or a file-like object of the image to create bmap for * bmap - full path or a file object to use for writing the resulting - bmap to """ + bmap to + """ self.image_size = None self.image_size_human = None @@ -160,8 +168,10 @@ class BmapCreate: self.blocks_cnt = self.fiemap.blocks_cnt def _bmap_file_start(self): - """ A helper function which generates the starting contents of the - block map file: the header comment, image size, block size, etc. """ + """ + A helper function which generates the starting contents of the block + map file: the header comment, image size, block size, etc. + """ # We do not know the amount of mapped blocks at the moment, so just put # whitespaces instead of real numbers. Assume the longest possible @@ -204,9 +214,11 @@ class BmapCreate: self._f_bmap.write(xml) def _bmap_file_end(self): - """ A helper function which generates the final parts of the block map + """ + A helper function which generates the final parts of the block map file: the ending tags and the information about the amount of mapped - blocks. """ + blocks. + """ xml = " \n" xml += "\n" @@ -226,8 +238,10 @@ class BmapCreate: self._f_bmap.write("%s" % sha1) def _calculate_sha1(self, first, last): - """ A helper function which calculates SHA1 checksum for the range of - blocks of the image file: from block 'first' to block 'last'. """ + """ + A helper function which calculates SHA1 checksum for the range of + blocks of the image file: from block 'first' to block 'last'. + """ start = first * self.block_size end = (last + 1) * self.block_size @@ -249,8 +263,10 @@ class BmapCreate: return hash_obj.hexdigest() def generate(self, include_checksums = True): - """ Generate bmap for the image file. If 'include_checksums' is 'True', - also generate SHA1 checksums for block ranges. """ + """ + Generate bmap for the image file. If 'include_checksums' is 'True', + also generate SHA1 checksums for block ranges. + """ # Save image file position in order to restore it at the end image_pos = self._f_image.tell() @@ -290,7 +306,7 @@ class BmapCreate: self._f_image.seek(image_pos) def __del__(self): - """ The class destructor which closes the opened files. """ + """The class destructor which closes the opened files.""" if self._f_image_needs_close: self._f_image.close() diff --git a/bmaptools/BmapHelpers.py b/bmaptools/BmapHelpers.py index 23204fb..ebda3dc 100644 --- a/bmaptools/BmapHelpers.py +++ b/bmaptools/BmapHelpers.py @@ -3,7 +3,7 @@ This module contains various shared helper functions. """ def human_size(size): - """ Transform size in bytes into a human-readable form. """ + """Transform size in bytes into a human-readable form.""" if size == 1: return "1 byte" @@ -19,7 +19,7 @@ def human_size(size): return "%.1f %s" % (size, 'EiB') def human_time(seconds): - """ Transform time in seconds to the HH:MM:SS format. """ + """Transform time in seconds to the HH:MM:SS format.""" (minutes, seconds) = divmod(seconds, 60) (hours, minutes) = divmod(minutes, 60) @@ -33,8 +33,10 @@ def human_time(seconds): return result + "%.1fs" % seconds def get_block_size(file_obj): - """ Returns block size for file object 'file_obj'. Errors are indicated by - the 'IOError' exception. """ + """ + Returns block size for file object 'file_obj'. Errors are indicated by the + 'IOError' exception. + """ from fcntl import ioctl import struct diff --git a/bmaptools/Fiemap.py b/bmaptools/Fiemap.py index 45670d5..0e2c9b5 100644 --- a/bmaptools/Fiemap.py +++ b/bmaptools/Fiemap.py @@ -1,5 +1,7 @@ -""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl -allows to find holes and mapped areas in a file. """ +""" +This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl +allows to find holes and mapped areas in a file. +""" # Note, a lot of code in this module is not very readable, because it deals # with the rather complex FIEMAP ioctl. To understand the code, you need to @@ -33,17 +35,21 @@ MIN_BUFFER_SIZE = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE DEFAULT_BUFFER_SIZE = 256 * 1024 class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ + """ + A class for exceptions generated by this module. We currently support only + one type of exceptions, and we basically throw human-readable problem + description in case of errors. + """ pass class Fiemap: - """ This class provides API to the FIEMAP ioctl. Namely, it allows to - iterate over all mapped blocks and over all holes. """ + """ + This class provides API to the FIEMAP ioctl. Namely, it allows to iterate + over all mapped blocks and over all holes. + """ def _open_image_file(self): - """ Open the image file. """ + """Open the image file.""" try: self._f_image = open(self._image_path, 'rb') @@ -54,13 +60,15 @@ class Fiemap: self._f_image_needs_close = True def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE): - """ Initialize a class instance. The 'image' argument is full path to - the file to operate on, or a file object to operate on. + """ + Initialize a class instance. The 'image' argument is full path to the + file to operate on, or a file object to operate on. The 'buf_size' argument is the size of the buffer for 'struct fiemap_extent' elements which will be used when invoking the FIEMAP ioctl. The larger is the buffer, the less times the FIEMAP ioctl will - be invoked. """ + be invoked. + """ self._f_image_needs_close = False @@ -112,18 +120,20 @@ class Fiemap: self.block_is_mapped(0) def __del__(self): - """ The class destructor which closes the opened files. """ + """The class destructor which closes the opened files.""" if self._f_image_needs_close: self._f_image.close() def _invoke_fiemap(self, block, count): - """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from + """ + Invoke the FIEMAP ioctl for 'count' blocks of the file starting from block number 'block'. The full result of the operation is stored in 'self._buf' on exit. Returns the unpacked 'struct fiemap' data structure in form of a python - list (just like 'struct.upack()'). """ + list (just like 'struct.upack()'). + """ if block < 0 or block >= self.blocks_cnt: raise Error("bad block number %d, should be within [0, %d]" \ @@ -148,8 +158,10 @@ class Fiemap: return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE]) def block_is_mapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is mapped and 'False' otherwise. """ + """ + This function returns 'True' if block number 'block' of the image file + is mapped and 'False' otherwise. + """ struct_fiemap = self._invoke_fiemap(block, 1) @@ -159,25 +171,31 @@ class Fiemap: return bool(struct_fiemap[3]) def block_is_unmapped(self, block): - """ This function returns 'True' if block number 'block' of the image - file is not mapped (hole) and 'False' otherwise. """ + """ + This function returns 'True' if block number 'block' of the image file + is not mapped (hole) and 'False' otherwise. + """ return not self.block_is_mapped(block) def _unpack_fiemap_extent(self, index): - """ Unpack a 'struct fiemap_extent' structure object number 'index' - from the internal 'self._buf' buffer. """ + """ + Unpack a 'struct fiemap_extent' structure object number 'index' from + the internal 'self._buf' buffer. + """ offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index return struct.unpack(_FIEMAP_EXTENT_FORMAT, self._buf[offset : offset + _FIEMAP_EXTENT_SIZE]) def _do_get_mapped_ranges(self, start, count): - """ Implements most the functionality for the 'get_mapped_ranges()' - generator: invokes the FIEMAP ioctl, walks through the mapped - extents and yields mapped block ranges. However, the ranges may be - consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' - simply merges them. """ + """ + Implements most the functionality for the 'get_mapped_ranges()' + generator: invokes the FIEMAP ioctl, walks through the mapped extents + and yields mapped block ranges. However, the ranges may be consecutive + (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges + them. + """ block = start while block < start + count: @@ -217,12 +235,14 @@ class Fiemap: block = extent_block + extent_count def get_mapped_ranges(self, start, count): - """ A generator which yields ranges of mapped blocks in the file. The + """ + A generator which yields ranges of mapped blocks in the file. The ranges are tuples of 2 elements: [first, last], where 'first' is the first mapped block and 'last' is the last mapped block. The ranges are yielded for the area of the file of size 'count' blocks, - starting from block 'start'. """ + starting from block 'start'. + """ iterator = self._do_get_mapped_ranges(start, count) @@ -238,8 +258,10 @@ class Fiemap: yield (first_prev, last_prev) def get_unmapped_ranges(self, start, count): - """ Just like 'get_mapped_ranges()', but yields unmapped block ranges - instead (holes). """ + """ + Just like 'get_mapped_ranges()', but yields unmapped block ranges + instead (holes). + """ hole_first = start for first, last in self._do_get_mapped_ranges(start, count): diff --git a/bmaptools/TransRead.py b/bmaptools/TransRead.py index 32e547a..0390110 100644 --- a/bmaptools/TransRead.py +++ b/bmaptools/TransRead.py @@ -1,7 +1,9 @@ -""" This module allows opening and reading local and remote files and -decompress them on-the-fly if needed. Remote files are read using urllib2 -(except of "ssh://" URLs, which are handled differently). Supported -compression types are: 'bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2'. """ +""" +This module allows opening and reading local and remote files and decompress +them on-the-fly if needed. Remote files are read using urllib2 (except of +"ssh://" URLs, which are handled differently). Supported compression types are: +'bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2'. +""" import os import errno @@ -19,9 +21,11 @@ import urlparse SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2') def _fake_seek_forward(file_obj, cur_pos, offset, whence = os.SEEK_SET): - """ This function implements the 'seek()' method for file object - 'file_obj'. Only seeking forward and is allowed, and 'whence' may be - either 'os.SEEK_SET' or 'os.SEEK_CUR'. """ + """ + This function implements the 'seek()' method for file object 'file_obj'. + Only seeking forward and is allowed, and 'whence' may be either + 'os.SEEK_SET' or 'os.SEEK_CUR'. + """ if whence == os.SEEK_SET: new_pos = offset @@ -53,19 +57,24 @@ def _fake_seek_forward(file_obj, cur_pos, offset, whence = os.SEEK_SET): return new_pos - to_read class Error(Exception): - """ A class for exceptions generated by this module. We currently support - only one type of exceptions, and we basically throw human-readable problem - description in case of errors. """ + """ + A class for exceptions generated by this module. We currently support only + one type of exceptions, and we basically throw human-readable problem + description in case of errors. + """ pass class _CompressedFile: - """ This class implements transparent reading from a compressed file-like - object and decompressing its contents on-the-fly. """ + """ + This class implements transparent reading from a compressed file-like + object and decompressing its contents on-the-fly. + """ def __init__(self, file_obj, decompress_func = None, chunk_size = None): - """ Class constructor. The 'file_ojb' argument is the compressed - file-like object to read from. The 'decompress_func()' function is a - function to use for decompression. + """ + Class constructor. The 'file_ojb' argument is the compressed file-like + object to read from. The 'decompress_func()' function is a function to + use for decompression. The 'chunk_size' parameter may be used to limit the amount of data read from the input file at a time and it is assumed to be used with @@ -77,7 +86,8 @@ class _CompressedFile: file filled with all zeroes is about 31MiB. Bzip2 is more dangerous - when 'chunk_size' is only 1KiB, the output buffer for a 4GiB .bz2 file filled with all zeroes is about 424MiB and when 'chunk_size' is 128 - bytes it is about 77MiB. """ + bytes it is about 77MiB. + """ self._file_obj = file_obj self._decompress_func = decompress_func @@ -91,17 +101,17 @@ class _CompressedFile: self._eof = False def seek(self, offset, whence = os.SEEK_SET): - """ The 'seek()' method, similar to the one file objects have. """ + """The 'seek()' method, similar to the one file objects have.""" self._pos = _fake_seek_forward(self, self._pos, offset, whence) def tell(self): - """ The 'tell()' method, similar to the one file objects have. """ + """The 'tell()' method, similar to the one file objects have.""" return self._pos def _read_from_buffer(self, length): - """ Read from the internal buffer. """ + """Read from the internal buffer.""" buffer_len = len(self._buffer) if buffer_len - self._buffer_pos > length: @@ -115,8 +125,10 @@ class _CompressedFile: return data def read(self, size): - """ Read the compressed file, uncompress the data on-the-fly, and - return 'size' bytes of the uncompressed data. """ + """ + Read the compressed file, uncompress the data on-the-fly, and return + 'size' bytes of the uncompressed data. + """ assert self._pos >= 0 assert self._buffer_pos >= 0 @@ -161,12 +173,14 @@ class _CompressedFile: return data def close(self): - """ Close the '_CompressedFile' file-like object. """ + """Close the '_CompressedFile' file-like object.""" pass def _decode_sshpass_exit_code(code): - """ A helper function which converts "sshpass" command-line tool's exit code - into a human-readable string. See "man sshpass". """ + """ + A helper function which converts "sshpass" command-line tool's exit code + into a human-readable string. See "man sshpass". + """ if code == 1: result = "invalid command line argument" @@ -190,15 +204,17 @@ def _decode_sshpass_exit_code(code): return result class TransRead: - """ This class implement the transparent reading functionality. Instances - of this class are file-like objects which you can read and seek only - forward. + """ + This class implement the transparent reading functionality. Instances of + this class are file-like objects which you can read and seek only forward. """ def _open_compressed_file(self): - """ Detect file compression type and open it with the corresponding + """ + Detect file compression type and open it with the corresponding compression module, or just plain 'open() if the file is not - compressed. """ + compressed. + """ try: if self.name.endswith('.tar.gz') \ @@ -232,9 +248,11 @@ class TransRead: raise Error("cannot open file '%s': %s" % (self.name, err)) def _open_url_ssh(self, url): - """ This function opens a file on a remote host using SSH. The URL has - to have this format: "ssh://username@hostname:path". Currently we only - support password-based authentication. """ + """ + This function opens a file on a remote host using SSH. The URL has to + have this format: "ssh://username@hostname:path". Currently we only + support password-based authentication. + """ import subprocess @@ -316,8 +334,9 @@ class TransRead: self._force_fake_seek = True def _open_url(self, url): - """ Open an URL 'url' and return the file-like object of the opened - URL. """ + """ + Open an URL 'url' and return the file-like object of the opened URL. + """ import urllib2 import httplib @@ -363,7 +382,7 @@ class TransRead: "status code that we don't understand" % url) def _create_local_copy(self): - """ Create a local copy of a remote or compressed file. """ + """Create a local copy of a remote or compressed file.""" import tempfile @@ -392,12 +411,14 @@ class TransRead: % (tmp_file_obj.name, err)) def __init__(self, filepath, local = False): - """ Class constructor. The 'filepath' argument is the full path to the - file to read transparently. If 'local' is True, then the file-like - object is guaranteed to be backed by a local file. This means that if - the source file is compressed or an URL, then it will first be copied - to a temporary local file, and then all the subsequent operations will - be done with the local copy. """ + """ + Class constructor. The 'filepath' argument is the full path to the file + to read transparently. If 'local' is True, then the file-like object is + guaranteed to be backed by a local file. This means that if the source + file is compressed or an URL, then it will first be copied to a + temporary local file, and then all the subsequent operations will be + done with the local copy. + """ self.name = filepath self.size = None @@ -424,8 +445,10 @@ class TransRead: self._create_local_copy() def read(self, size = -1): - """ Read the data from the file or URL and and uncompress it on-the-fly - if necessary. """ + """ + Read the data from the file or URL and and uncompress it on-the-fly if + necessary. + """ if size < 0: size = 0xFFFFFFFFFFFFFFFF @@ -436,7 +459,7 @@ class TransRead: return buf def __del__(self): - """ The class destructor which closes opened files. """ + """The class destructor which closes opened files.""" if self._transfile_obj: self._transfile_obj.close() @@ -446,7 +469,7 @@ class TransRead: self._child_process.wait() def seek(self, offset, whence = os.SEEK_SET): - """ The 'seek()' method, similar to the one file objects have. """ + """The 'seek()' method, similar to the one file objects have.""" if self._force_fake_seek or not hasattr(self._transfile_obj, "seek"): self._pos = _fake_seek_forward(self._transfile_obj, self._pos, \ @@ -455,7 +478,7 @@ class TransRead: self._transfile_obj.seek(offset, whence) def tell(self): - """ The 'tell()' method, similar to the one file objects have. """ + """The 'tell()' method, similar to the one file objects have.""" if self._force_fake_seek or not hasattr(self._transfile_obj, "tell"): return self._pos @@ -463,13 +486,15 @@ class TransRead: return self._transfile_obj.tell() def close(self): - """ Close the file-like object. """ + """Close the file-like object.""" self.__del__() def __getattr__(self, name): - """ If we are backed by a local uncompressed file, then fall-back to - using its operations. """ + """ + If we are backed by a local uncompressed file, then fall-back to using + its operations. + """ if not self.is_compressed and not self.is_url: return getattr(self._transfile_obj, name) -- 2.7.4