import logging
import socket
+from select import poll, POLLIN
+from socket import error as SocketError, timeout as SocketTimeout
+
+
try: # Python 3
from http.client import HTTPConnection, HTTPSConnection, HTTPException
+ from http.client import HTTP_PORT, HTTPS_PORT
except ImportError:
from httplib import HTTPConnection, HTTPSConnection, HTTPException
+ from httplib import HTTP_PORT, HTTPS_PORT
try: # Python 3
from queue import Queue, Empty, Full
except ImportError:
from Queue import Queue, Empty, Full
-
-from select import select
-from socket import error as SocketError, timeout as SocketTimeout
-
-from .packages.ssl_match_hostname import match_hostname, CertificateError
-try:
+try: # Compiled with SSL?
import ssl
BaseSSLError = ssl.SSLError
except ImportError:
from .request import RequestMethods
from .response import HTTPResponse
-from .exceptions import (
- SSLError,
+from .exceptions import (SSLError,
MaxRetryError,
TimeoutError,
HostChangedError,
EmptyPoolError,
)
-from . import six
-xrange = six.moves.xrange
+from urllib3.packages.ssl_match_hostname import match_hostname, CertificateError
+from urllib3.packages import six
+xrange = six.moves.xrange
log = logging.getLogger(__name__)
_Default = object()
+port_by_scheme = {
+ 'http': HTTP_PORT,
+ 'https': HTTPS_PORT,
+}
## Connection objects (extension of httplib)
Base class for all connection pools, such as
:class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`.
"""
- pass
+
+ scheme = None
+
+ def __init__(self, host, port=None):
+ self.host = host
+ self.port = port
+
+ def __str__(self):
+ return '%s(host=%r, port=%r)' % (type(self).__name__,
+ self.host, self.port)
class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.pool.get(block=self.block, timeout=timeout)
# If this is a persistent connection, check if it got disconnected
- if conn and conn.sock and select([conn.sock], [], [], 0.0)[0]:
- # Either data is buffered (bad), or the connection is dropped.
+ if conn and conn.sock and is_connection_dropped(conn):
log.info("Resetting dropped connection: %s" % self.host)
conn.close()
except Empty:
if self.block:
- raise EmptyPoolError("Pool reached maximum size and no more "
+ raise EmptyPoolError(self,
+ "Pool reached maximum size and no more "
"connections are allowed.")
pass # Oh well, we'll create a new connection then
def is_same_host(self, url):
"""
Check if the given ``url`` is a member of the same host as this
- conncetion pool.
+ connection pool.
"""
# TODO: Add optional support for socket.gethostbyname checking.
+ scheme, host, port = get_host(url)
+
+ if self.port and not port:
+ # Use explicit default port for comparison when none is given.
+ port = port_by_scheme.get(scheme)
+
return (url.startswith('/') or
- get_host(url) == (self.scheme, self.host, self.port))
+ (scheme, host, port) == (self.scheme, self.host, self.port))
def urlopen(self, method, url, body=None, headers=None, retries=3,
redirect=True, assert_same_host=True, timeout=_Default,
headers = self.headers
if retries < 0:
- raise MaxRetryError(url)
+ raise MaxRetryError(self, url)
if timeout is _Default:
timeout = self.timeout
if self.port:
host = "%s:%d" % (host, self.port)
- raise HostChangedError(host, url, retries - 1)
+ raise HostChangedError(self, url, retries - 1)
conn = None
except Empty as e:
# Timed out by queue
- raise TimeoutError("Request timed out. (pool_timeout=%s)" %
+ raise TimeoutError(self, "Request timed out. (pool_timeout=%s)" %
pool_timeout)
except SocketTimeout as e:
# Timed out by socket
- raise TimeoutError("Request timed out. (timeout=%s)" %
+ raise TimeoutError(self, "Request timed out. (timeout=%s)" %
timeout)
except BaseSSLError as e:
return HTTPSConnectionPool(host, port=port, **kw)
else:
return HTTPConnectionPool(host, port=port, **kw)
+
+
+def is_connection_dropped(conn):
+ """
+ Returns True if the connection is dropped and should be closed.
+
+ :param conn:
+ ``HTTPConnection`` object.
+ """
+ # poll-based replacement to select([conn.sock], [], [], 0.0)[0]:
+ p = poll()
+ p.register(conn.sock, POLLIN)
+ for (fno, ev) in p.poll(0.0):
+ if fno == conn.sock.fileno():
+ # Either data is buffered (bad), or the connection is dropped.
+ return True
# This module is part of urllib3 and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
-## Exceptions
+## Base Exceptions
class HTTPError(Exception):
"Base exception used by this module."
pass
-class SSLError(Exception):
+class PoolError(HTTPError):
+ "Base exception for errors caused within a pool."
+ def __init__(self, pool, message):
+ self.pool = pool
+ HTTPError.__init__(self, "%s: %s" % (pool, message))
+
+
+class SSLError(HTTPError):
"Raised when SSL certificate fails in an HTTPS connection."
pass
-class MaxRetryError(HTTPError):
- "Raised when the maximum number of retries is exceeded."
- def __init__(self, url):
- HTTPError.__init__(self, "Max retries exceeded for url: %s" % url)
- self.url = url
+## Leaf Exceptions
+class MaxRetryError(PoolError):
+ "Raised when the maximum number of retries is exceeded."
+ def __init__(self, pool, url):
+ PoolError.__init__(self, pool,
+ "Max retries exceeded with url: %s" % url)
-class TimeoutError(HTTPError):
- "Raised when a socket timeout occurs."
- pass
+ self.url = url
-class HostChangedError(HTTPError):
+class HostChangedError(PoolError):
"Raised when an existing pool gets a request for a foreign host."
- def __init__(self, original_host, new_url, retries=3):
- HTTPError.__init__(self,
- "Connection pool with host '%s' tried to open a foreign host: %s" %
- (original_host, new_url))
+ def __init__(self, pool, url, retries=3):
+ PoolError.__init__(self, pool,
+ "Tried to open a foreign host with url: %s" % url)
- self.original_host = original_host
- self.new_url = new_url
+ self.url = url
self.retries = retries
-class EmptyPoolError(HTTPError):
+class TimeoutError(PoolError):
+ "Raised when a socket timeout occurs."
+ pass
+
+
+class EmptyPoolError(PoolError):
"Raised when a pool runs out of connections and no more are allowed."
pass
try:
from mimetools import choose_boundary
except ImportError:
- # I don't like using an undocumented function, but I don't yet know what it does
- from email.generator import _make_boundary as choose_boundary
+ from .packages.mimetools_choose_boundary import choose_boundary
from io import BytesIO
-from . import six
-from .six import b
+from .packages import six
+from .packages.six import b
writer = codecs.lookup('utf-8')[3]
--- /dev/null
+"""The function mimetools.choose_boundary() from Python 2.7, which seems to
+have disappeared in Python 3 (although email.generator._make_boundary() might
+work as a replacement?).
+
+Tweaked to use lock from threading rather than thread.
+"""
+import os
+from threading import Lock
+_counter_lock = Lock()
+
+_counter = 0
+def _get_next_counter():
+ global _counter
+ with _counter_lock:
+ _counter += 1
+ return _counter
+
+_prefix = None
+
+def choose_boundary():
+ """Return a string usable as a multipart boundary.
+
+ The string chosen is unique within a single program run, and
+ incorporates the user id (if available), process id (if available),
+ and current time. So it's very unlikely the returned string appears
+ in message text, but there's no guarantee.
+
+ The boundary contains dots so you have to quote it in the header."""
+
+ global _prefix
+ import time
+ if _prefix is None:
+ import socket
+ try:
+ hostid = socket.gethostbyname(socket.gethostname())
+ except socket.gaierror:
+ hostid = '127.0.0.1'
+ try:
+ uid = repr(os.getuid())
+ except AttributeError:
+ uid = '1'
+ try:
+ pid = repr(os.getpid())
+ except AttributeError:
+ pid = '1'
+ _prefix = hostid + '.' + uid + '.' + pid
+ return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter())
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
-from .connectionpool import get_host, connection_from_url
+from .connectionpool import get_host, connection_from_url, port_by_scheme
from .exceptions import HostChangedError
from .request import RequestMethods
'https': HTTPSConnectionPool,
}
-port_by_scheme = {
- 'http': 80,
- 'https': 443,
-}
-
log = logging.getLogger(__name__)
except HostChangedError as e:
kw['retries'] = e.retries # Persist retries countdown
- return self.urlopen(method, e.new_url, **kw)
+ return self.urlopen(method, e.url, **kw)
class ProxyManager(RequestMethods):
self.strict = strict
self._decode_content = decode_content
- self._body = None
+ self._body = body if body and isinstance(body, basestring) else None
self._fp = None
self._original_response = original_response
if hasattr(body, 'read'):
self._fp = body
- if preload_content:
+ if preload_content and not self._body:
self._body = self.read(decode_content=decode_content)
def get_redirect_location(self):
if decode_content is None:
decode_content = self._decode_content
- data = self._fp and self._fp.read(amt)
+ if self._fp is None:
+ return
try:
-
- if amt:
- return data
-
- if not decode_content or not decoder:
- if cache_content:
- self._body = data
-
- return data
+ if amt is None:
+ # cStringIO doesn't like amt=None
+ data = self._fp.read()
+ else:
+ return self._fp.read(amt)
try:
- data = decoder(data)
+ if decode_content and decoder:
+ data = decoder(data)
except IOError:
raise HTTPError("Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding)
return data
finally:
-
if self._original_response and self._original_response.isclosed():
self.release_conn()
projects / services / python-requests.git / commitdiff