From 7cc2d468c7e4620a32c829e2ff0117b3395cf6ab Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Sat, 25 Feb 2012 23:11:20 -0500 Subject: [PATCH] urllib3 v1.2.2 --- requests/packages/urllib3/__init__.py | 2 +- requests/packages/urllib3/connectionpool.py | 55 +++++++++++++++++++++-------- requests/packages/urllib3/exceptions.py | 21 ++++++++--- requests/packages/urllib3/poolmanager.py | 10 ++++++ requests/packages/urllib3/response.py | 7 +--- 5 files changed, 69 insertions(+), 26 deletions(-) diff --git a/requests/packages/urllib3/__init__.py b/requests/packages/urllib3/__init__.py index 5f70c56..2e9c663 100644 --- a/requests/packages/urllib3/__init__.py +++ b/requests/packages/urllib3/__init__.py @@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using. __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.1' +__version__ = '1.2.2' from .connectionpool import ( diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 52b1802..39e652e 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -7,6 +7,7 @@ import logging import socket +from base64 import b64encode from socket import error as SocketError, timeout as SocketTimeout try: @@ -16,33 +17,45 @@ except ImportError: # Doesn't exist on OSX and other platforms poll = False try: # Python 3 - from http.client import HTTPConnection, HTTPSConnection, HTTPException + from http.client import HTTPConnection, HTTPException from http.client import HTTP_PORT, HTTPS_PORT except ImportError: - from httplib import HTTPConnection, HTTPSConnection, HTTPException + from httplib import HTTPConnection, HTTPException from httplib import HTTP_PORT, HTTPS_PORT try: # Python 3 - from queue import Queue, Empty, Full + from queue import LifoQueue, Empty, Full except ImportError: - from Queue import Queue, Empty, Full + from Queue import LifoQueue, Empty, Full + try: # Compiled with SSL? + HTTPSConnection = object + BaseSSLError = None + ssl = None + + try: # Python 3 + from http.client import HTTPSConnection + except ImportError: + from httplib import HTTPSConnection + import ssl BaseSSLError = ssl.SSLError + except ImportError: - ssl = None - BaseSSLError = None + pass from .packages.ssl_match_hostname import match_hostname, CertificateError from .request import RequestMethods from .response import HTTPResponse -from .exceptions import (SSLError, +from .exceptions import ( + EmptyPoolError, + HostChangedError, + LocationParseError, MaxRetryError, + SSLError, TimeoutError, - HostChangedError, - EmptyPoolError, ) from .packages.ssl_match_hostname import match_hostname, CertificateError @@ -103,6 +116,7 @@ class ConnectionPool(object): """ scheme = None + QueueCls = LifoQueue def __init__(self, host, port=None): self.host = host @@ -156,11 +170,11 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): def __init__(self, host, port=None, strict=False, timeout=None, maxsize=1, block=False, headers=None): - self.host = host - self.port = port + super(HTTPConnectionPool, self).__init__(host, port) + self.strict = strict self.timeout = timeout - self.pool = Queue(maxsize) + self.pool = self.QueueCls(maxsize) self.block = block self.headers = headers or {} @@ -468,7 +482,11 @@ class HTTPSConnectionPool(HTTPConnectionPool): log.info("Starting new HTTPS connection (%d): %s" % (self.num_connections, self.host)) - if not ssl: + if not ssl: # Platform-specific: Python compiled without +ssl + if not HTTPSConnection or HTTPSConnection is object: + raise SSLError("Can't connect to HTTPS URL because the SSL " + "module is not available.") + return HTTPSConnection(host=self.host, port=self.port) connection = VerifiedHTTPSConnection(host=self.host, port=self.port) @@ -526,7 +544,7 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, if basic_auth: headers['authorization'] = 'Basic ' + \ - basic_auth.encode('base64').strip() + b64encode(six.b(basic_auth)).decode('utf-8') return headers @@ -542,10 +560,12 @@ def get_host(url): >>> get_host('google.com:80') ('http', 'google.com', 80) """ + # This code is actually similar to urlparse.urlsplit, but much # simplified for our needs. port = None scheme = 'http' + if '://' in url: scheme, url = url.split('://', 1) if '/' in url: @@ -554,7 +574,12 @@ def get_host(url): _auth, url = url.split('@', 1) if ':' in url: url, port = url.split(':', 1) + + if not port.isdigit(): + raise LocationParseError("Failed to parse: %s") + port = int(port) + return scheme, url, port @@ -592,7 +617,7 @@ def is_connection_dropped(conn): :param conn: ``HTTPConnection`` object. """ - if not poll: + if not poll: # Platform-specific return select([conn.sock], [], [], 0.0)[0] # This version is better on platforms that support it. diff --git a/requests/packages/urllib3/exceptions.py b/requests/packages/urllib3/exceptions.py index 0bffeb4..15c9699 100644 --- a/requests/packages/urllib3/exceptions.py +++ b/requests/packages/urllib3/exceptions.py @@ -4,6 +4,7 @@ # This module is part of urllib3 and is released under # the MIT License: http://www.opensource.org/licenses/mit-license.php + ## Base Exceptions class HTTPError(Exception): @@ -27,18 +28,20 @@ class SSLError(HTTPError): class MaxRetryError(PoolError): "Raised when the maximum number of retries is exceeded." + def __init__(self, pool, url): - PoolError.__init__(self, pool, - "Max retries exceeded with url: %s" % url) + message = "Max retries exceeded with url: %s" % url + PoolError.__init__(self, pool, message) self.url = url class HostChangedError(PoolError): "Raised when an existing pool gets a request for a foreign host." + def __init__(self, pool, url, retries=3): - PoolError.__init__(self, pool, - "Tried to open a foreign host with url: %s" % url) + message = "Tried to open a foreign host with url: %s" % url + PoolError.__init__(self, pool, message) self.url = url self.retries = retries @@ -52,3 +55,13 @@ class TimeoutError(PoolError): class EmptyPoolError(PoolError): "Raised when a pool runs out of connections and no more are allowed." pass + + +class LocationParseError(ValueError, HTTPError): + "Raised when get_host or similar fails to parse the URL input." + + def __init__(self, location): + message = "Failed to parse: %s" % location + super(LocationParseError, self).__init__(self, message) + + self.location = location diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index f194b2e..d42f35b 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -117,9 +117,19 @@ class ProxyManager(RequestMethods): def __init__(self, proxy_pool): self.proxy_pool = proxy_pool + def _set_proxy_headers(self, headers=None): + headers = headers or {} + + # Same headers are curl passes for --proxy1.0 + headers['Accept'] = '*/*' + headers['Proxy-Connection'] = 'Keep-Alive' + + return headers + def urlopen(self, method, url, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." kw['assert_same_host'] = False + kw['headers'] = self._set_proxy_headers(kw.get('headers')) return self.proxy_pool.urlopen(method, url, **kw) diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index e023970..4dd431e 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -11,12 +11,7 @@ import zlib from io import BytesIO from .exceptions import HTTPError - - -try: - basestring = basestring -except NameError: # Python 3 - basestring = (str, bytes) +from .packages.six import string_types as basestring log = logging.getLogger(__name__) -- 2.7.4