From 2b849545ea61f216e0699ad8454d24c609d0e3a4 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 7 Jan 2012 17:18:49 -0500 Subject: [PATCH] urllib3 update --- requests/packages/urllib3/connectionpool.py | 22 +++++------- requests/packages/urllib3/exceptions.py | 14 ++++++-- requests/packages/urllib3/poolmanager.py | 23 +++++++----- requests/packages/urllib3/response.py | 40 ++++++++++++++------- 4 files changed, 62 insertions(+), 37 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index c1ebed4..17f2f84 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -306,7 +306,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): headers = self.headers if retries < 0: - raise MaxRetryError("Max retries exceeded for url: %s" % url) + raise MaxRetryError(url) if timeout is _Default: timeout = self.timeout @@ -320,8 +320,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if self.port: host = "%s:%d" % (host, self.port) - raise HostChangedError("Connection pool with host '%s' tried to " - "open a foreign host: %s" % (host, url)) + raise HostChangedError(host, url, retries - 1) conn = None @@ -369,7 +368,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except (CertificateError), e: # Name mismatch raise SSLError(e) - + except (HTTPException, SocketError), e: # Connection broken, discard. It will be replaced next _get_conn(). conn = None @@ -385,15 +384,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return self.urlopen(method, url, body, headers, retries - 1, redirect, assert_same_host) # Try again - # Handle redirection - if (redirect and - response.status in [301, 302, 303, 307] and - 'location' in response.headers): # Redirect, retry - log.info("Redirecting %s -> %s" % - (url, response.headers.get('location'))) - return self.urlopen(method, response.headers.get('location'), body, - headers, retries - 1, redirect, - assert_same_host) + # Handle redirect? + redirect_location = redirect and response.get_redirect_location() + if redirect_location: + log.info("Redirecting %s -> %s" % (url, redirect_location)) + return self.urlopen(method, redirect_location, body, headers, + retries - 1, redirect, assert_same_host) return response diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 69f459b..47937f7 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -18,7 +18,9 @@ class SSLError(Exception): class MaxRetryError(HTTPError): "Raised when the maximum number of retries is exceeded." - pass + def __init__(self, url): + HTTPError.__init__(self, "Max retries exceeded for url: %s" % url) + self.url = url class TimeoutError(HTTPError): @@ -28,7 +30,15 @@ class TimeoutError(HTTPError): class HostChangedError(HTTPError): "Raised when an existing pool gets a request for a foreign host." - pass + def __init__(self, original_host, new_url, retries=3): + HTTPError.__init__(self, + "Connection pool with host '%s' tried to open a foreign host: %s" % + (original_host, new_url)) + + self.original_host = original_host + self.new_url = new_url + self.retries = retries + class EmptyPoolError(HTTPError): "Raised when a pool runs out of connections and no more are allowed." diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index c08e327..482ee4a 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -4,20 +4,18 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php +import logging + from ._collections import RecentlyUsedContainer -from .connectionpool import ( - HTTPConnectionPool, HTTPSConnectionPool, - get_host, connection_from_url, -) +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool +from .connectionpool import get_host, connection_from_url +from .exceptions import HostChangedError +from .request import RequestMethods __all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] -from .request import RequestMethods -from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool - - pool_classes_by_scheme = { 'http': HTTPConnectionPool, 'https': HTTPSConnectionPool, @@ -28,6 +26,8 @@ port_by_scheme = { 'https': 443, } +log = logging.getLogger(__name__) + class PoolManager(RequestMethods): """ @@ -105,7 +105,12 @@ class PoolManager(RequestMethods): :class:`urllib3.connectionpool.ConnectionPool` can be chosen for it. """ conn = self.connection_from_url(url) - return conn.urlopen(method, url, assert_same_host=False, **kw) + try: + return conn.urlopen(method, url, **kw) + + except HostChangedError, e: + kw['retries'] = e.retries # Persist retries countdown + return self.urlopen(method, e.new_url, **kw) class ProxyManager(RequestMethods): diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 4cd15c1..ee2ff66 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -84,6 +84,19 @@ class HTTPResponse(object): if preload_content: self._body = self.read(decode_content=decode_content) + def get_redirect_location(self): + """ + Should we redirect and where to? + + :returns: Truthy redirect location string if we got a redirect status + code and valid location. ``None`` if redirect status and no + location. ``False`` if not a redirect status code. + """ + if self.status in [301, 302, 303, 307]: + return self.headers.get('location') + + return False + def release_conn(self): if not self._pool or not self._connection: return @@ -98,10 +111,9 @@ class HTTPResponse(object): return self._body if self._fp: - return self.read(decode_content=self._decode_content, - cache_content=True) + return self.read(cache_content=True) - def read(self, amt=None, decode_content=True, cache_content=False): + def read(self, amt=None, decode_content=None, cache_content=False): """ Similar to :meth:`httplib.HTTPResponse.read`, but with two additional parameters: ``decode_content`` and ``cache_content``. @@ -124,6 +136,8 @@ class HTTPResponse(object): """ content_encoding = self.headers.get('content-encoding') decoder = self.CONTENT_DECODERS.get(content_encoding) + if decode_content is None: + decode_content = self._decode_content data = self._fp and self._fp.read(amt) @@ -154,8 +168,8 @@ class HTTPResponse(object): if self._original_response and self._original_response.isclosed(): self.release_conn() - @staticmethod - def from_httplib(r, **response_kw): + @classmethod + def from_httplib(ResponseCls, r, **response_kw): """ Given an :class:`httplib.HTTPResponse` instance ``r``, return a corresponding :class:`urllib3.response.HTTPResponse` object. @@ -164,14 +178,14 @@ class HTTPResponse(object): with ``original_response=r``. """ - return HTTPResponse(body=r, - headers=dict(r.getheaders()), - status=r.status, - version=r.version, - reason=r.reason, - strict=r.strict, - original_response=r, - **response_kw) + return ResponseCls(body=r, + headers=dict(r.getheaders()), + status=r.status, + version=r.version, + reason=r.reason, + strict=r.strict, + original_response=r, + **response_kw) # Backwards-compatibility methods for httplib.HTTPResponse def getheaders(self): -- 2.34.1