From 6e53290acd02085424c548c6e149c3811325d314 Mon Sep 17 00:00:00 2001 From: Mantas Vidutis Date: Tue, 22 Jan 2013 14:40:10 -0800 Subject: [PATCH] update urllib3 to current master with ssl bugfixes --- requests/packages/urllib3/connectionpool.py | 26 ++++++--------- requests/packages/urllib3/poolmanager.py | 17 +++++++--- requests/packages/urllib3/response.py | 12 ++++++- requests/packages/urllib3/util.py | 52 +++++++++++++++++++++++++---- 4 files changed, 81 insertions(+), 26 deletions(-) diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index af8760d..51c87f5 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -9,6 +9,7 @@ import socket import errno from socket import error as SocketError, timeout as SocketTimeout +from .util import resolve_cert_reqs, resolve_ssl_version try: # Python 3 from http.client import HTTPConnection, HTTPException @@ -80,31 +81,29 @@ class VerifiedHTTPSConnection(HTTPSConnection): ssl_version = None def set_cert(self, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None): - ssl_req_scheme = { - 'CERT_NONE': ssl.CERT_NONE, - 'CERT_OPTIONAL': ssl.CERT_OPTIONAL, - 'CERT_REQUIRED': ssl.CERT_REQUIRED - } + cert_reqs=None, ca_certs=None): self.key_file = key_file self.cert_file = cert_file - self.cert_reqs = ssl_req_scheme.get(cert_reqs) or ssl.CERT_NONE + self.cert_reqs = cert_reqs self.ca_certs = ca_certs def connect(self): # Add certificate verification sock = socket.create_connection((self.host, self.port), self.timeout) + resolved_cert_reqs = resolve_cert_reqs(self.cert_reqs) + resolved_ssl_version = resolve_ssl_version(self.ssl_version) + # Wrap socket using verification with the root certs in # trusted_root_certs self.sock = ssl_wrap_socket(sock, self.key_file, self.cert_file, - cert_reqs=self.cert_reqs, + cert_reqs=resolved_cert_reqs, ca_certs=self.ca_certs, server_hostname=self.host, - ssl_version=self.ssl_version) + ssl_version=resolved_ssl_version) - if self.ca_certs: + if resolved_cert_reqs != ssl.CERT_NONE: match_hostname(self.sock.getpeercert(), self.host) @@ -514,7 +513,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): strict=False, timeout=None, maxsize=1, block=False, headers=None, key_file=None, cert_file=None, - cert_reqs='CERT_NONE', ca_certs=None, ssl_version=None): + cert_reqs=None, ca_certs=None, ssl_version=None): HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, @@ -548,10 +547,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): connection.set_cert(key_file=self.key_file, cert_file=self.cert_file, cert_reqs=self.cert_reqs, ca_certs=self.ca_certs) - if self.ssl_version is None: - connection.ssl_version = ssl.PROTOCOL_SSLv23 - else: - connection.ssl_version = self.ssl_version + connection.ssl_version = self.ssl_version return connection diff --git a/requests/packages/urllib3/poolmanager.py b/requests/packages/urllib3/poolmanager.py index a124202..d58f9f7 100644 --- a/requests/packages/urllib3/poolmanager.py +++ b/requests/packages/urllib3/poolmanager.py @@ -58,6 +58,17 @@ class PoolManager(RequestMethods): self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + def _new_pool(self, scheme, host, port): + """ + Create a new :class:`ConnectionPool` based on host, port and scheme. + + This method is used to actually create the connection pools handed out + by :meth:`connection_from_url` and companion methods. It is intended + to be overridden for customization. + """ + pool_cls = pool_classes_by_scheme[scheme] + return pool_cls(host, port, **self.connection_pool_kw) + def clear(self): """ Empty our store of pools and direct them all to close. @@ -74,6 +85,7 @@ class PoolManager(RequestMethods): If ``port`` isn't given, it will be derived from the ``scheme`` using ``urllib3.connectionpool.port_by_scheme``. """ + scheme = scheme or 'http' port = port or port_by_scheme.get(scheme, 80) pool_key = (scheme, host, port) @@ -85,11 +97,8 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - pool_cls = pool_classes_by_scheme[scheme] - pool = pool_cls(host, port, **self.connection_pool_kw) - + pool = self._new_pool(scheme, host, port) self.pools[pool_key] = pool - return pool def connection_from_url(self, url): diff --git a/requests/packages/urllib3/response.py b/requests/packages/urllib3/response.py index 833be62..0761dc0 100644 --- a/requests/packages/urllib3/response.py +++ b/requests/packages/urllib3/response.py @@ -145,7 +145,17 @@ class HTTPResponse(object): # cStringIO doesn't like amt=None data = self._fp.read() else: - return self._fp.read(amt) + data = self._fp.read(amt) + if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + # Close the connection when no data is returned + # + # This is redundant to what httplib/http.client _should_ + # already do. However, versions of python released before + # December 15, 2012 (http://bugs.python.org/issue16298) do not + # properly close the connection in all cases. There is no harm + # in redundantly calling close. + self._fp.close() + return data try: if decode_content and decoder: diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py index 8d8654f..b827bc4 100644 --- a/requests/packages/urllib3/util.py +++ b/requests/packages/urllib3/util.py @@ -22,6 +22,7 @@ try: # Test for SSL features SSLContext = None HAS_SNI = False + import ssl from ssl import wrap_socket, CERT_NONE, SSLError, PROTOCOL_SSLv23 from ssl import SSLContext # Modern SSL? from ssl import HAS_SNI # Has SNI? @@ -263,10 +264,48 @@ def is_connection_dropped(conn): return True +def resolve_cert_reqs(candidate): + """ + Resolves the argument to a numeric constant, which can be passed to + the wrap_socket function/method from the ssl module. + Defaults to :data:`ssl.CERT_NONE`. + If given a string it is assumed to be the name of the constant in the + :mod:`ssl` module or its abbrevation. + (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. + If it's neither `None` nor a string we assume it is already the numeric + constant which can directly be passed to wrap_socket. + """ + if candidate is None: + return CERT_NONE + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'CERT_' + candidate) + return res + + return candidate + + +def resolve_ssl_version(candidate): + """ + like resolve_cert_reqs + """ + if candidate is None: + return PROTOCOL_SSLv23 + + if isinstance(candidate, str): + res = getattr(ssl, candidate, None) + if res is None: + res = getattr(ssl, 'PROTOCOL_' + candidate) + return res + + return candidate + if SSLContext is not None: # Python 3.2+ - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): """ All arguments except `server_hostname` have the same meaning as for :func:`ssl.wrap_socket` @@ -279,8 +318,9 @@ if SSLContext is not None: # Python 3.2+ if ca_certs: try: context.load_verify_locations(ca_certs) - except TypeError as e: # Reraise as SSLError - # FIXME: This block needs a test. + # Py32 raises IOError + # Py33 raises FileNotFoundError + except Exception as e: # Reraise as SSLError raise SSLError(e) if certfile: # FIXME: This block needs a test. @@ -290,9 +330,9 @@ if SSLContext is not None: # Python 3.2+ return context.wrap_socket(sock) else: # Python 3.1 and earlier - def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=CERT_NONE, + def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, ca_certs=None, server_hostname=None, - ssl_version=PROTOCOL_SSLv23): + ssl_version=None): return wrap_socket(sock, keyfile=keyfile, certfile=certfile, ca_certs=ca_certs, cert_reqs=cert_reqs, ssl_version=ssl_version) -- 2.7.4