Use streaming decompression feature of urllib3.

diff --git a/requests/models.py b/requests/models.py
index 105287751d0335d3f8a88da692ab0d0c8c9f1223..4cf69b57d7b8bed1c2c385d4a1178ae9823315da 100644 (file)
--- a/requests/models.py
+++ b/requests/models.py
@@ -20,7 +20,7 @@ from .cookies import cookiejar_from_dict, get_cookie_header
 from .packages.urllib3.filepost import encode_multipart_formdata
 from .exceptions import HTTPError, RequestException, MissingSchema, InvalidURL
 from .utils import (
-    stream_untransfer, guess_filename, get_auth_from_url, requote_uri,
+    guess_filename, get_auth_from_url, requote_uri,
     stream_decode_response_unicode, to_key_val_list, parse_header_links,
     iter_slices, guess_json_utf, super_len)
 from .compat import (
@@ -528,13 +528,13 @@ class Response(object):
 
         def generate():
             while 1:
-                chunk = self.raw.read(chunk_size)
+                chunk = self.raw.read(chunk_size, decode_content=True)
                 if not chunk:
                     break
                 yield chunk
             self._content_consumed = True
 
-        gen = stream_untransfer(generate(), self)
+        gen = generate()
 
         if decode_unicode:
             gen = stream_decode_response_unicode(gen, self)

diff --git a/requests/utils.py b/requests/utils.py
index a2d434effdb6d8fa291abeef27b3d7a43b91db0a..68f3e625aac0acb6b7c017fa704d17f412b5baf4 100644 (file)
--- a/requests/utils.py
+++ b/requests/utils.py
@@ -346,48 +346,6 @@ def get_unicode_from_response(r):
         return r.content
 
 
-def stream_decompress(iterator, mode='gzip'):
-    """Stream decodes an iterator over compressed data
-
-    :param iterator: An iterator over compressed data
-    :param mode: 'gzip' or 'deflate'
-    :return: An iterator over decompressed data
-    """
-
-    if mode not in ['gzip', 'deflate']:
-        raise ValueError('stream_decompress mode must be gzip or deflate')
-
-    zlib_mode = 16 + zlib.MAX_WBITS if mode == 'gzip' else -zlib.MAX_WBITS
-    dec = zlib.decompressobj(zlib_mode)
-    try:
-        for chunk in iterator:
-            rv = dec.decompress(chunk)
-            if rv:
-                yield rv
-    except zlib.error:
-        # If there was an error decompressing, just return the raw chunk
-        yield chunk
-        # Continue to return the rest of the raw data
-        for chunk in iterator:
-            yield chunk
-    else:
-        # Make sure everything has been returned from the decompression object
-        buf = dec.decompress(bytes())
-        rv = buf + dec.flush()
-        if rv:
-            yield rv
-
-
-def stream_untransfer(gen, resp):
-    ce = resp.headers.get('content-encoding', '').lower()
-    if 'gzip' in ce:
-        gen = stream_decompress(gen, mode='gzip')
-    elif 'deflate' in ce:
-        gen = stream_decompress(gen, mode='deflate')
-
-    return gen
-
-
 # The unreserved URI characters (RFC 3986)
 UNRESERVED_SET = frozenset(
     "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"