# License: GPLv2
# Author: Artem Bityutskiy <artem.bityutskiy@linux.intel.com>
-""" This is a tool to generate block map files (bmap) and to copy files using
-bmap. Generally speaking, these tools are about writing large image files
-quickly.
+"""
+This is a tool to generate block map files (bmap) and to copy files using bmap.
+Generally speaking, these tools are about writing large image files quickly.
The bmap file is an XML file which contains a list of mapped blocks of the
image. Mapped blocks are the blocks which have disk sectors associated with
stick (namely, you write only mapped blocks). This is a lot faster than copying
all 4GiB of data. We say that it is a bit more than 100MiB because things like
file-system meta-data (inode tables, superblocks, etc), partition table, etc
-also contribute to the mapped blocks and are also copied. """
+also contribute to the mapped blocks and are also copied.
+"""
# Disable the following pylint recommendations:
# * Too few public methods (R0903)
from bmaptools import BmapCreate, BmapCopy, BmapHelpers, TransRead
def copy_command_open_blkdev(path, log):
- """ Open a block device specified by 'path' in exclusive mode. Returns
- opened file object. """
+ """
+ Open a block device specified by 'path' in exclusive mode. Returns opened
+ file object.
+ """
class NamedFile:
- """ This simple class allows us to override the 'name' attribute of a
- file object. The problem is that 'os.fdopen()' sets the name to
- "<fdopen>", which is not very user-friendly. """
+ """
+ This simple class allows us to override the 'name' attribute of a file
+ object. The problem is that 'os.fdopen()' sets the name to "<fdopen>",
+ which is not very user-friendly.
+ """
def __init__(self, file_obj, name):
self._file_obj = file_obj
return NamedFile(file_obj, path)
def find_and_open_bmap(image_path):
- """ When the user does not specify the bmap file, we try to find it at the
- same place where the image file is located. We search for a file with the
- same path and basename, but with a ".bmap" extension. Since the image may
+ """
+ When the user does not specify the bmap file, we try to find it at the same
+ place where the image file is located. We search for a file with the same
+ path and basename, but with a ".bmap" extension. Since the image may
contain more than one extension (e.g., image.raw.bz2), try to remove them
one-by-one.
This function returns a file-like object for the bmap file if it has been
- found, and 'None' otherwise. """
+ found, and 'None' otherwise.
+ """
bmap_path = None
return None
def copy_command_open_all(args, log):
- """ Open the image/bmap/destination files for the "copy" command. Returns a
+ """
+ Open the image/bmap/destination files for the "copy" command. Returns a
tuple of 5 elements:
1 file-like object for the image
2 file object for the destination file
def copy_command(args, log):
- """ Copy an image to a block device or a regular file using bmap. """
+ """Copy an image to a block device or a regular file using bmap."""
image_obj, dest_obj, bmap_obj, bmap_path, image_size, dest_is_blkdev = \
copy_command_open_all(args, log)
image_obj.close()
def create_command(args, log):
- """ Generate block map (AKA bmap) for an image. The idea is that while
- images files may generally be very large (e.g., 4GiB), they may
- nevertheless contain only little real data, e.g., 512MiB. This data are
- files, directories, file-system meta-data, partition table, etc. When
- copying the image to the target device, you do not have to copy all the
- 4GiB of data, you can copy only 512MiB of it, which is 4 times less, so
- copying should presumably be 4 times faster.
+ """
+ Generate block map (AKA bmap) for an image. The idea is that while images
+ files may generally be very large (e.g., 4GiB), they may nevertheless
+ contain only little real data, e.g., 512MiB. This data are files,
+ directories, file-system meta-data, partition table, etc. When copying the
+ image to the target device, you do not have to copy all the 4GiB of data,
+ you can copy only 512MiB of it, which is 4 times less, so copying should
+ presumably be 4 times faster.
The block map file is an XML file which contains a list of blocks which
have to be copied to the target device. The other blocks are not used and
or parts of it), etc. The end result should be a sparse file where mapped
areas represent useful parts of the image and holes represent useless parts
of the image, which do not have to be copied when copying the image to the
- target device. """
+ target device.
+ """
# Create and setup the output stream
if args.output:
"were expanded?")
def parse_arguments():
- """ A helper function which parses the input arguments. """
+ """A helper function which parses the input arguments."""
text = "Create block map (bmap) and copy files using bmap. The " \
"documentation can be found here: " \
return parser.parse_args()
def setup_logger(loglevel):
- """ A helper function which sets up and configures the logger. The log
- level is initialized to 'loglevel'. Returns the logger object. """
+ """
+ A helper function which sets up and configures the logger. The log level is
+ initialized to 'loglevel'. Returns the logger object.
+ """
# Esc-sequences for coloured output
esc_red = '\033[91m' # pylint: disable=W1401
return log
def main():
- """ Script entry point. """
+ """Script entry point."""
args = parse_arguments()
-""" This module implements copying of images with bmap and provides the
-following API.
+"""
+This module implements copying of images with bmap and provides the following
+API.
1. BmapCopy class - implements copying to any kind of file, be that a block
device or a regular file.
2. BmapBdevCopy class - based on BmapCopy and specializes on copying to block
stick (namely, you copy only mapped blocks). This is a lot faster than copying
all 4GiB of data. We say that it is a bit more than 100MiB because things like
file-system meta-data (inode tables, superblocks, etc), partition table, etc
-also contribute to the mapped blocks and are also copied. """
+also contribute to the mapped blocks and are also copied.
+"""
# Disable the following pylint recommendations:
# * Too many instance attributes (R0902)
SUPPORTED_BMAP_VERSION = 1
class Error(Exception):
- """ A class for exceptions generated by the 'BmapCopy' module. We currently
+ """
+ A class for exceptions generated by the 'BmapCopy' module. We currently
support only one type of exceptions, and we basically throw human-readable
- problem description in case of errors. """
+ problem description in case of errors.
+ """
pass
class BmapCopy:
- """ This class implements the bmap-based copying functionality. To copy an
+ """
+ This class implements the bmap-based copying functionality. To copy an
image with bmap you should create an instance of this class, which requires
the following:
You can copy only once with an instance of this class. This means that in
order to copy the image for the second time, you have to create a new class
- instance. """
+ instance.
+ """
def set_progress_indicator(self, file_obj, format_string):
- """ Setup the progress indicator which shows how much data has been
- copied in percent.
+ """
+ Setup the progress indicator which shows how much data has been copied
+ in percent.
The 'file_obj' argument is the console file object where the progress
has to be printed to. Pass 'None' to disable the progress indicator.
The 'format_string' argument is the format string for the progress
indicator. It has to contain a single '%d' placeholder which will be
- substitutes with copied data in percent. """
+ substitutes with copied data in percent.
+ """
self._progress_file = file_obj
if format_string:
self._progress_format = "Copied %d%%"
def _set_image_size(self, image_size):
- """ Set image size and initialize various other geometry-related
- attributes. """
+ """
+ Set image size and initialize various other geometry-related attributes.
+ """
if self.image_size is not None and self.image_size != image_size:
raise Error("cannot set image size to %d bytes, it is known to " \
self.mapped_size_human = self.image_size_human
def _verify_bmap_checksum(self):
- """ This is a helper function which verifies SHA1 checksum of the bmap
- file. """
+ """
+ This is a helper function which verifies SHA1 checksum of the bmap file.
+ """
import mmap
(self._bmap_path, calculated_sha1, correct_sha1))
def _parse_bmap(self):
- """ Parse the bmap file and initialize corresponding class instance
- attributs. """
+ """
+ Parse the bmap file and initialize corresponding class instance attributs.
+ """
try:
self._xml = ElementTree.parse(self._f_bmap)
def __init__(self, image, dest, bmap = None, image_size = None,
logger = None):
- """ The class constructor. The parameters are:
+ """
+ The class constructor. The parameters are:
image - file-like object of the image which should be copied,
should only support 'read()' and 'seek()' methods,
and only seeking forward has to be supported.
bmap - file object of the bmap file to use for copying.
image_size - size of the image in bytes.
logger - the logger object to use for printing messages.
- """
+ """
self._logger = logger
if self._logger is None:
self._batch_blocks = self._batch_bytes / self.block_size
def _update_progress(self, blocks_written):
- """ Print the progress indicator if the mapped area size is known and
- if the indicator has been enabled by assigning a console file object to
- the 'progress_file' attribute. """
+ """
+ Print the progress indicator if the mapped area size is known and if
+ the indicator has been enabled by assigning a console file object to
+ the 'progress_file' attribute.
+ """
if not self._progress_file:
return
self._progress_file.flush()
def _get_block_ranges(self):
- """ This is a helper generator that parses the bmap XML file and for
- each block range in the XML file it yields ('first', 'last', 'sha1')
- tuples, where:
+ """
+ This is a helper generator that parses the bmap XML file and for each
+ block range in the XML file it yields ('first', 'last', 'sha1') tuples,
+ where:
* 'first' is the first block of the range;
* 'last' is the last block of the range;
* 'sha1' is the SHA1 checksum of the range ('None' is used if it is
If there is no bmap file, the generator just yields a single range
for entire image file. If the image size is unknown, the generator
- infinitely yields continuous ranges of size '_batch_blocks'. """
+ infinitely yields continuous ranges of size '_batch_blocks'.
+ """
if not self._f_bmap:
# We do not have the bmap, yield a tuple with all blocks
yield (first, last, sha1)
def _get_batches(self, first, last):
- """ This is a helper generator which splits block ranges from the bmap
- file to smaller batches. Indeed, we cannot read and write entire block
+ """
+ This is a helper generator which splits block ranges from the bmap file
+ to smaller batches. Indeed, we cannot read and write entire block
ranges from the image file, because a range can be very large. So we
perform the I/O in batches. Batch size is defined by the
'_batch_blocks' attribute. Thus, for each (first, last) block range,
* 'start' is the starting batch block number;
* 'last' is the ending batch block number;
* 'length' is the batch length in blocks (same as
- 'end' - 'start' + 1). """
+ 'end' - 'start' + 1).
+ """
batch_blocks = self._batch_blocks
yield (first, first + batch_blocks - 1, batch_blocks)
def _get_data(self, verify):
- """ This is generator which reads the image file in '_batch_blocks'
- chunks and yields ('type', 'start', 'end', 'buf) tuples, where:
+ """
+ This is generator which reads the image file in '_batch_blocks' chunks
+ and yields ('type', 'start', 'end', 'buf) tuples, where:
* 'start' is the starting block number of the batch;
* 'end' is the last block of the batch;
- * 'buf' a buffer containing the batch data. """
+ * 'buf' a buffer containing the batch data.
+ """
try:
for (first, last, sha1) in self._get_block_ranges():
self._batch_queue.put(None)
def copy(self, sync = True, verify = True):
- """ Copy the image to the destination file using bmap. The 'sync'
- argument defines whether the destination file has to be synchronized
- upon return. The 'verify' argument defines whether the SHA1 checksum
- has to be verified while copying. """
+ """
+ Copy the image to the destination file using bmap. The 'sync' argument
+ defines whether the destination file has to be synchronized upon
+ return. The 'verify' argument defines whether the SHA1 checksum has to
+ be verified while copying.
+ """
# Create the queue for block batches and start the reader thread, which
# will read the image in batches and put the results to '_batch_queue'.
self.sync()
def sync(self):
- """ Synchronize the destination file to make sure all the data are
- actually written to the disk. """
+ """
+ Synchronize the destination file to make sure all the data are actually
+ written to the disk.
+ """
if self._dest_supports_fsync:
try:
class BmapBdevCopy(BmapCopy):
- """ This class is a specialized version of 'BmapCopy' which copies the
- image to a block device. Unlike the base 'BmapCopy' class, this class does
- various optimizations specific to block devices, e.g., switching to the
- 'noop' I/O scheduler. """
+ """
+ This class is a specialized version of 'BmapCopy' which copies the image to
+ a block device. Unlike the base 'BmapCopy' class, this class does various
+ optimizations specific to block devices, e.g., switching to the 'noop' I/O
+ scheduler.
+ """
def _tune_block_device(self):
- """" Tune the block device for better performance:
+ """
+ Tune the block device for better performance:
1. Switch to the 'noop' I/O scheduler if it is available - sequential
write to the block device becomes a lot faster comparing to CFQ.
2. Limit the write buffering - we do not need the kernel to buffer a
"I/O scheduler: %s)" % err)
def _restore_bdev_settings(self):
- """ Restore old block device settings which we changed in
- '_tune_block_device()'. """
+ """
+ Restore old block device settings which we changed in
+ '_tune_block_device()'.
+ """
if self._old_scheduler_value is not None:
try:
% (self._old_max_ratio_value, err))
def copy(self, sync = True, verify = True):
- """ The same as in the base class but tunes the block device for better
- performance before starting writing. Additionally, it forces block
- device synchronization from time to time in order to make sure we do
+ """
+ The same as in the base class but tunes the block device for better performance before starting writing. Additionally, it forces block device synchronization from time to time in order to make sure we do
not get stuck in 'fsync()' for too long time. The problem is that the
kernel synchronizes block devices when the file is closed. And the
result is that if the user interrupts us while we are copying the data,
the program will be blocked in 'close()' waiting for the block device
synchronization, which may last minutes for slow USB stick. This is
very bad user experience, and we work around this effect by
- synchronizing from time to time. """
+ synchronizing from time to time.
+ """
self._tune_block_device()
def __init__(self, image, dest, bmap = None, image_size = None,
logger = None):
- """ The same as the constructor of the 'BmapCopy' base class, but adds
- useful guard-checks specific to block devices. """
+ """
+ The same as the constructor of the 'BmapCopy' base class, but adds
+ useful guard-checks specific to block devices.
+ """
# Call the base class constructor first
BmapCopy.__init__(self, image, dest, bmap, image_size, logger=logger)
-""" This module implements the block map (bmap) creation functionality and
-provides the corresponding API in form of the 'BmapCreate' class.
+"""
+This module implements the block map (bmap) creation functionality and provides
+the corresponding API in form of the 'BmapCreate' class.
The idea is that while images files may generally be very large (e.g., 4GiB),
they may nevertheless contain only little real data, e.g., 512MiB. This data
parts of the image and holes represent useless parts of the image, which do not
have to be copied when copying the image to the target device.
-This module uses the FIBMAP ioctl to detect holes. """
+This module uses the FIBMAP ioctl to detect holes.
+"""
# Disable the following pylint recommendations:
# * Too many instance attributes - R0902
"""
class Error(Exception):
- """ A class for exceptions generated by this module. We currently support
- only one type of exceptions, and we basically throw human-readable problem
- description in case of errors. """
+ """
+ A class for exceptions generated by this module. We currently support only
+ one type of exceptions, and we basically throw human-readable problem
+ description in case of errors.
+ """
pass
class BmapCreate:
- """ This class implements the bmap creation functionality. To generate a
- bmap for an image (which is supposedly a sparse file), you should first
- create an instance of 'BmapCreate' and provide:
+ """
+ This class implements the bmap creation functionality. To generate a bmap
+ for an image (which is supposedly a sparse file), you should first create
+ an instance of 'BmapCreate' and provide:
* full path or a file-like object of the image to create bmap for
* full path or a file object to use for writing the results to
Then you should invoke the 'generate()' method of this class. It will use
- the FIEMAP ioctl to generate the bmap. """
+ the FIEMAP ioctl to generate the bmap.
+ """
def _open_image_file(self):
- """ Open the image file. """
+ """Open the image file."""
try:
self._f_image = open(self._image_path, 'rb')
self._f_image_needs_close = True
def _open_bmap_file(self):
- """ Open the bmap file. """
+ """Open the bmap file."""
try:
self._f_bmap = open(self._bmap_path, 'w+')
self._f_bmap_needs_close = True
def __init__(self, image, bmap):
- """ Initialize a class instance:
+ """
+ Initialize a class instance:
* image - full path or a file-like object of the image to create bmap
for
* bmap - full path or a file object to use for writing the resulting
- bmap to """
+ bmap to
+ """
self.image_size = None
self.image_size_human = None
self.blocks_cnt = self.fiemap.blocks_cnt
def _bmap_file_start(self):
- """ A helper function which generates the starting contents of the
- block map file: the header comment, image size, block size, etc. """
+ """
+ A helper function which generates the starting contents of the block
+ map file: the header comment, image size, block size, etc.
+ """
# We do not know the amount of mapped blocks at the moment, so just put
# whitespaces instead of real numbers. Assume the longest possible
self._f_bmap.write(xml)
def _bmap_file_end(self):
- """ A helper function which generates the final parts of the block map
+ """
+ A helper function which generates the final parts of the block map
file: the ending tags and the information about the amount of mapped
- blocks. """
+ blocks.
+ """
xml = " </BlockMap>\n"
xml += "</bmap>\n"
self._f_bmap.write("%s" % sha1)
def _calculate_sha1(self, first, last):
- """ A helper function which calculates SHA1 checksum for the range of
- blocks of the image file: from block 'first' to block 'last'. """
+ """
+ A helper function which calculates SHA1 checksum for the range of
+ blocks of the image file: from block 'first' to block 'last'.
+ """
start = first * self.block_size
end = (last + 1) * self.block_size
return hash_obj.hexdigest()
def generate(self, include_checksums = True):
- """ Generate bmap for the image file. If 'include_checksums' is 'True',
- also generate SHA1 checksums for block ranges. """
+ """
+ Generate bmap for the image file. If 'include_checksums' is 'True',
+ also generate SHA1 checksums for block ranges.
+ """
# Save image file position in order to restore it at the end
image_pos = self._f_image.tell()
self._f_image.seek(image_pos)
def __del__(self):
- """ The class destructor which closes the opened files. """
+ """The class destructor which closes the opened files."""
if self._f_image_needs_close:
self._f_image.close()
"""
def human_size(size):
- """ Transform size in bytes into a human-readable form. """
+ """Transform size in bytes into a human-readable form."""
if size == 1:
return "1 byte"
return "%.1f %s" % (size, 'EiB')
def human_time(seconds):
- """ Transform time in seconds to the HH:MM:SS format. """
+ """Transform time in seconds to the HH:MM:SS format."""
(minutes, seconds) = divmod(seconds, 60)
(hours, minutes) = divmod(minutes, 60)
return result + "%.1fs" % seconds
def get_block_size(file_obj):
- """ Returns block size for file object 'file_obj'. Errors are indicated by
- the 'IOError' exception. """
+ """
+ Returns block size for file object 'file_obj'. Errors are indicated by the
+ 'IOError' exception.
+ """
from fcntl import ioctl
import struct
-""" This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl
-allows to find holes and mapped areas in a file. """
+"""
+This module implements python API for the FIEMAP ioctl. The FIEMAP ioctl
+allows to find holes and mapped areas in a file.
+"""
# Note, a lot of code in this module is not very readable, because it deals
# with the rather complex FIEMAP ioctl. To understand the code, you need to
DEFAULT_BUFFER_SIZE = 256 * 1024
class Error(Exception):
- """ A class for exceptions generated by this module. We currently support
- only one type of exceptions, and we basically throw human-readable problem
- description in case of errors. """
+ """
+ A class for exceptions generated by this module. We currently support only
+ one type of exceptions, and we basically throw human-readable problem
+ description in case of errors.
+ """
pass
class Fiemap:
- """ This class provides API to the FIEMAP ioctl. Namely, it allows to
- iterate over all mapped blocks and over all holes. """
+ """
+ This class provides API to the FIEMAP ioctl. Namely, it allows to iterate
+ over all mapped blocks and over all holes.
+ """
def _open_image_file(self):
- """ Open the image file. """
+ """Open the image file."""
try:
self._f_image = open(self._image_path, 'rb')
self._f_image_needs_close = True
def __init__(self, image, buf_size = DEFAULT_BUFFER_SIZE):
- """ Initialize a class instance. The 'image' argument is full path to
- the file to operate on, or a file object to operate on.
+ """
+ Initialize a class instance. The 'image' argument is full path to the
+ file to operate on, or a file object to operate on.
The 'buf_size' argument is the size of the buffer for 'struct
fiemap_extent' elements which will be used when invoking the FIEMAP
ioctl. The larger is the buffer, the less times the FIEMAP ioctl will
- be invoked. """
+ be invoked.
+ """
self._f_image_needs_close = False
self.block_is_mapped(0)
def __del__(self):
- """ The class destructor which closes the opened files. """
+ """The class destructor which closes the opened files."""
if self._f_image_needs_close:
self._f_image.close()
def _invoke_fiemap(self, block, count):
- """ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
+ """
+ Invoke the FIEMAP ioctl for 'count' blocks of the file starting from
block number 'block'.
The full result of the operation is stored in 'self._buf' on exit.
Returns the unpacked 'struct fiemap' data structure in form of a python
- list (just like 'struct.upack()'). """
+ list (just like 'struct.upack()').
+ """
if block < 0 or block >= self.blocks_cnt:
raise Error("bad block number %d, should be within [0, %d]" \
return struct.unpack(_FIEMAP_FORMAT, self._buf[:_FIEMAP_SIZE])
def block_is_mapped(self, block):
- """ This function returns 'True' if block number 'block' of the image
- file is mapped and 'False' otherwise. """
+ """
+ This function returns 'True' if block number 'block' of the image file
+ is mapped and 'False' otherwise.
+ """
struct_fiemap = self._invoke_fiemap(block, 1)
return bool(struct_fiemap[3])
def block_is_unmapped(self, block):
- """ This function returns 'True' if block number 'block' of the image
- file is not mapped (hole) and 'False' otherwise. """
+ """
+ This function returns 'True' if block number 'block' of the image file
+ is not mapped (hole) and 'False' otherwise.
+ """
return not self.block_is_mapped(block)
def _unpack_fiemap_extent(self, index):
- """ Unpack a 'struct fiemap_extent' structure object number 'index'
- from the internal 'self._buf' buffer. """
+ """
+ Unpack a 'struct fiemap_extent' structure object number 'index' from
+ the internal 'self._buf' buffer.
+ """
offset = _FIEMAP_SIZE + _FIEMAP_EXTENT_SIZE * index
return struct.unpack(_FIEMAP_EXTENT_FORMAT,
self._buf[offset : offset + _FIEMAP_EXTENT_SIZE])
def _do_get_mapped_ranges(self, start, count):
- """ Implements most the functionality for the 'get_mapped_ranges()'
- generator: invokes the FIEMAP ioctl, walks through the mapped
- extents and yields mapped block ranges. However, the ranges may be
- consecutive (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()'
- simply merges them. """
+ """
+ Implements most the functionality for the 'get_mapped_ranges()'
+ generator: invokes the FIEMAP ioctl, walks through the mapped extents
+ and yields mapped block ranges. However, the ranges may be consecutive
+ (e.g., (1, 100), (100, 200)) and 'get_mapped_ranges()' simply merges
+ them.
+ """
block = start
while block < start + count:
block = extent_block + extent_count
def get_mapped_ranges(self, start, count):
- """ A generator which yields ranges of mapped blocks in the file. The
+ """
+ A generator which yields ranges of mapped blocks in the file. The
ranges are tuples of 2 elements: [first, last], where 'first' is the
first mapped block and 'last' is the last mapped block.
The ranges are yielded for the area of the file of size 'count' blocks,
- starting from block 'start'. """
+ starting from block 'start'.
+ """
iterator = self._do_get_mapped_ranges(start, count)
yield (first_prev, last_prev)
def get_unmapped_ranges(self, start, count):
- """ Just like 'get_mapped_ranges()', but yields unmapped block ranges
- instead (holes). """
+ """
+ Just like 'get_mapped_ranges()', but yields unmapped block ranges
+ instead (holes).
+ """
hole_first = start
for first, last in self._do_get_mapped_ranges(start, count):
-""" This module allows opening and reading local and remote files and
-decompress them on-the-fly if needed. Remote files are read using urllib2
-(except of "ssh://" URLs, which are handled differently). Supported
-compression types are: 'bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2'. """
+"""
+This module allows opening and reading local and remote files and decompress
+them on-the-fly if needed. Remote files are read using urllib2 (except of
+"ssh://" URLs, which are handled differently). Supported compression types are:
+'bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2'.
+"""
import os
import errno
SUPPORTED_COMPRESSION_TYPES = ('bz2', 'gz', 'tar.gz', 'tgz', 'tar.bz2')
def _fake_seek_forward(file_obj, cur_pos, offset, whence = os.SEEK_SET):
- """ This function implements the 'seek()' method for file object
- 'file_obj'. Only seeking forward and is allowed, and 'whence' may be
- either 'os.SEEK_SET' or 'os.SEEK_CUR'. """
+ """
+ This function implements the 'seek()' method for file object 'file_obj'.
+ Only seeking forward and is allowed, and 'whence' may be either
+ 'os.SEEK_SET' or 'os.SEEK_CUR'.
+ """
if whence == os.SEEK_SET:
new_pos = offset
return new_pos - to_read
class Error(Exception):
- """ A class for exceptions generated by this module. We currently support
- only one type of exceptions, and we basically throw human-readable problem
- description in case of errors. """
+ """
+ A class for exceptions generated by this module. We currently support only
+ one type of exceptions, and we basically throw human-readable problem
+ description in case of errors.
+ """
pass
class _CompressedFile:
- """ This class implements transparent reading from a compressed file-like
- object and decompressing its contents on-the-fly. """
+ """
+ This class implements transparent reading from a compressed file-like
+ object and decompressing its contents on-the-fly.
+ """
def __init__(self, file_obj, decompress_func = None, chunk_size = None):
- """ Class constructor. The 'file_ojb' argument is the compressed
- file-like object to read from. The 'decompress_func()' function is a
- function to use for decompression.
+ """
+ Class constructor. The 'file_ojb' argument is the compressed file-like
+ object to read from. The 'decompress_func()' function is a function to
+ use for decompression.
The 'chunk_size' parameter may be used to limit the amount of data read
from the input file at a time and it is assumed to be used with
file filled with all zeroes is about 31MiB. Bzip2 is more dangerous -
when 'chunk_size' is only 1KiB, the output buffer for a 4GiB .bz2 file
filled with all zeroes is about 424MiB and when 'chunk_size' is 128
- bytes it is about 77MiB. """
+ bytes it is about 77MiB.
+ """
self._file_obj = file_obj
self._decompress_func = decompress_func
self._eof = False
def seek(self, offset, whence = os.SEEK_SET):
- """ The 'seek()' method, similar to the one file objects have. """
+ """The 'seek()' method, similar to the one file objects have."""
self._pos = _fake_seek_forward(self, self._pos, offset, whence)
def tell(self):
- """ The 'tell()' method, similar to the one file objects have. """
+ """The 'tell()' method, similar to the one file objects have."""
return self._pos
def _read_from_buffer(self, length):
- """ Read from the internal buffer. """
+ """Read from the internal buffer."""
buffer_len = len(self._buffer)
if buffer_len - self._buffer_pos > length:
return data
def read(self, size):
- """ Read the compressed file, uncompress the data on-the-fly, and
- return 'size' bytes of the uncompressed data. """
+ """
+ Read the compressed file, uncompress the data on-the-fly, and return
+ 'size' bytes of the uncompressed data.
+ """
assert self._pos >= 0
assert self._buffer_pos >= 0
return data
def close(self):
- """ Close the '_CompressedFile' file-like object. """
+ """Close the '_CompressedFile' file-like object."""
pass
def _decode_sshpass_exit_code(code):
- """ A helper function which converts "sshpass" command-line tool's exit code
- into a human-readable string. See "man sshpass". """
+ """
+ A helper function which converts "sshpass" command-line tool's exit code
+ into a human-readable string. See "man sshpass".
+ """
if code == 1:
result = "invalid command line argument"
return result
class TransRead:
- """ This class implement the transparent reading functionality. Instances
- of this class are file-like objects which you can read and seek only
- forward.
+ """
+ This class implement the transparent reading functionality. Instances of
+ this class are file-like objects which you can read and seek only forward.
"""
def _open_compressed_file(self):
- """ Detect file compression type and open it with the corresponding
+ """
+ Detect file compression type and open it with the corresponding
compression module, or just plain 'open() if the file is not
- compressed. """
+ compressed.
+ """
try:
if self.name.endswith('.tar.gz') \
raise Error("cannot open file '%s': %s" % (self.name, err))
def _open_url_ssh(self, url):
- """ This function opens a file on a remote host using SSH. The URL has
- to have this format: "ssh://username@hostname:path". Currently we only
- support password-based authentication. """
+ """
+ This function opens a file on a remote host using SSH. The URL has to
+ have this format: "ssh://username@hostname:path". Currently we only
+ support password-based authentication.
+ """
import subprocess
self._force_fake_seek = True
def _open_url(self, url):
- """ Open an URL 'url' and return the file-like object of the opened
- URL. """
+ """
+ Open an URL 'url' and return the file-like object of the opened URL.
+ """
import urllib2
import httplib
"status code that we don't understand" % url)
def _create_local_copy(self):
- """ Create a local copy of a remote or compressed file. """
+ """Create a local copy of a remote or compressed file."""
import tempfile
% (tmp_file_obj.name, err))
def __init__(self, filepath, local = False):
- """ Class constructor. The 'filepath' argument is the full path to the
- file to read transparently. If 'local' is True, then the file-like
- object is guaranteed to be backed by a local file. This means that if
- the source file is compressed or an URL, then it will first be copied
- to a temporary local file, and then all the subsequent operations will
- be done with the local copy. """
+ """
+ Class constructor. The 'filepath' argument is the full path to the file
+ to read transparently. If 'local' is True, then the file-like object is
+ guaranteed to be backed by a local file. This means that if the source
+ file is compressed or an URL, then it will first be copied to a
+ temporary local file, and then all the subsequent operations will be
+ done with the local copy.
+ """
self.name = filepath
self.size = None
self._create_local_copy()
def read(self, size = -1):
- """ Read the data from the file or URL and and uncompress it on-the-fly
- if necessary. """
+ """
+ Read the data from the file or URL and and uncompress it on-the-fly if
+ necessary.
+ """
if size < 0:
size = 0xFFFFFFFFFFFFFFFF
return buf
def __del__(self):
- """ The class destructor which closes opened files. """
+ """The class destructor which closes opened files."""
if self._transfile_obj:
self._transfile_obj.close()
self._child_process.wait()
def seek(self, offset, whence = os.SEEK_SET):
- """ The 'seek()' method, similar to the one file objects have. """
+ """The 'seek()' method, similar to the one file objects have."""
if self._force_fake_seek or not hasattr(self._transfile_obj, "seek"):
self._pos = _fake_seek_forward(self._transfile_obj, self._pos, \
self._transfile_obj.seek(offset, whence)
def tell(self):
- """ The 'tell()' method, similar to the one file objects have. """
+ """The 'tell()' method, similar to the one file objects have."""
if self._force_fake_seek or not hasattr(self._transfile_obj, "tell"):
return self._pos
return self._transfile_obj.tell()
def close(self):
- """ Close the file-like object. """
+ """Close the file-like object."""
self.__del__()
def __getattr__(self, name):
- """ If we are backed by a local uncompressed file, then fall-back to
- using its operations. """
+ """
+ If we are backed by a local uncompressed file, then fall-back to using
+ its operations.
+ """
if not self.is_compressed and not self.is_url:
return getattr(self._transfile_obj, name)
projects / tools / bmap-tools.git / commitdiff