BmapCopy: implement threaded reader

Read the data from a separate thread. This does not change the performance
measurably in case of uncompressed images, but does improve writing speed in
case of compressed images - Tizen.bz2 image flashing drops from 2m13s to
1m43s.

Change-Id: I3300a569042d16357fcc0920c3f03f3fff08d29d
Signed-off-by: Artem Bityutskiy <artem.bityutskiy@intel.com>

diff --git a/TODO b/TODO
index 7cc954f..ef6aa87 100644 (file)
--- a/TODO
+++ b/TODO
@@ -5,7 +5,5 @@
 3. In 'bmaptool create' when the blocks rage consists of a single block (say
    18282), write it in form of <>18282</> instead of less readable
    <>18282-18282</> form. 'bmaptool copy' supports both already.
-4. Try to read and write in parrallell from different threads - this may improve
-   USB stick flashing speed.
-5. The block device optimizations will not work if it was /dev/sdc1 instead of
+4. The block device optimizations will not work if it was /dev/sdc1 instead of
    /dev/sdc - fix this.

diff --git a/bmaptools/BmapCopy.py b/bmaptools/BmapCopy.py
index 0e49a0c..3b1b65e 100644 (file)
--- a/bmaptools/BmapCopy.py
+++ b/bmaptools/BmapCopy.py
@@ -33,6 +33,8 @@ also contribute to the mapped blocks and are also copied. """
 import os
 import stat
 import hashlib
+import Queue
+import thread
 from xml.etree import ElementTree
 from bmaptools.BmapHelpers import human_size
 
@@ -205,7 +207,9 @@ class BmapCopy:
 
         self._dest_fsync_watermark = None
         self._batch_blocks = None
+        self._batch_queue = None
         self._batch_bytes = 1024 * 1024
+        self._batch_queue_len = 2
 
         self.bmap_version = None
         self.bmap_block_size = None
@@ -372,13 +376,15 @@ class BmapCopy:
                 if verify and sha1:
                     hash_obj.update(buf)
 
-                yield (start, end, length, buf)
+                self._batch_queue.put((start, end, length, buf))
 
             if verify and sha1 and hash_obj.hexdigest() != sha1:
                 raise Error("checksum mismatch for blocks range %d-%d: " \
                             "calculated %s, should be %s" \
                             % (first, last, hash_obj.hexdigest(), sha1))
 
+        self._batch_queue.put(None)
+
     def copy(self, sync = True, verify = True):
         """ Copy the image to the destination file using bmap. The sync
         argument defines whether the destination file has to be synchronized
@@ -389,12 +395,24 @@ class BmapCopy:
             self._copy_entire_image(sync)
             return
 
+        # Create the queue for block batches and start the reader thread, which
+        # will read the image in batches and put the results to '_batch_queue'.
+        self._batch_queue = Queue.Queue(self._batch_queue_len)
+        thread.start_new_thread(self._get_data, (verify, ))
+
         blocks_written = 0
         fsync_last = 0
 
         # Read the image in '_batch_blocks' chunks and write them to the
         # destination file
-        for (start, end, length, buf) in self._get_data(verify):
+        while True:
+            batch = self._batch_queue.get()
+            if batch is None:
+                # No more data, the image is written
+                break
+
+            (start, end, length, buf) = batch
+
             self._f_dest.seek(start * self.bmap_block_size)
 
             # Synchronize the destination file if we reached the watermark
@@ -409,6 +427,7 @@ class BmapCopy:
                 raise Error("error while writing blocks %d-%d of '%s': %s" \
                             % (start, end, self._dest_path, err))
 
+            self._batch_queue.task_done()
             blocks_written += length
 
         # This is just a sanity check - we should have written exactly
@@ -521,8 +540,6 @@ class BmapBdevCopy(BmapCopy):
         synchronizing from time to time. """
 
         self._tune_block_device()
-        self._dest_fsync_watermark = (6 * 1024 * 1024) / self.bmap_block_size
-
         BmapCopy.copy(self, sync, verify)
 
     def __init__(self, image_path, dest_path, bmap_path = None):
@@ -532,6 +549,11 @@ class BmapBdevCopy(BmapCopy):
         # Call the base class construcor first
         BmapCopy.__init__(self, image_path, dest_path, bmap_path)
 
+        self._batch_bytes = 1024 * 1024
+        self._batch_blocks = self._batch_bytes / self.bmap_block_size
+        self._batch_queue_len = 6
+        self._dest_fsync_watermark = (6 * 1024 * 1024) / self.bmap_block_size
+
         # If the image size is known (i.e., it is not compressed) - check that
         # itfits the block device.
         if self.bmap_image_size: