log = logging.getLogger(__name__)
+_Default = object()
+
## Connection objects (extension of httplib)
log.warning("HttpConnectionPool is full, discarding connection: %s"
% self.host)
- def _make_request(self, conn, method, url, **httplib_request_kw):
+ def _make_request(self, conn, method, url, timeout=_Default,
+ **httplib_request_kw):
"""
Perform a request on a given httplib connection object taken from our
pool.
"""
self.num_requests += 1
+ if timeout is _Default:
+ timeout = self.timeout
+
conn.request(method, url, **httplib_request_kw)
- conn.sock.settimeout(self.timeout)
+ conn.sock.settimeout(timeout)
httplib_response = conn.getresponse()
log.debug("\"%s %s %s\" %s %s" %
get_host(url) == (self.scheme, self.host, self.port))
def urlopen(self, method, url, body=None, headers=None, retries=3,
- redirect=True, assert_same_host=True, pool_timeout=None,
- release_conn=None, **response_kw):
+ redirect=True, assert_same_host=True, timeout=_Default,
+ pool_timeout=None, release_conn=None, **response_kw):
"""
Get a connection from the pool and perform an HTTP request.
consistent else will raise HostChangedError. When False, you can
use the pool on an HTTP proxy and request foreign hosts.
+ timeout
+ If specified, overrides the default timeout for this one request.
+
pool_timeout
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
try:
# Make the request on the httplib connection object
httplib_response = self._make_request(conn, method, url,
+ timeout=timeout,
body=body, headers=headers)
# Import httplib's response into our own wrapper object
response = HTTPResponse.from_httplib(httplib_response,
self.pools = RecentlyUsedContainer(num_pools)
self.recently_used_pools = []
- def connection_from_url(self, url):
+ def connection_from_host(self, host, port=80, scheme='http'):
"""
- Similar to connectionpool.connection_from_url but doesn't pass any
- additional keywords to the ConnectionPool constructor. Additional
- keywords are taken from the PoolManager constructor.
+ Get a ConnectionPool based on the host, port, and scheme.
"""
- scheme, host, port = get_host(url)
+ pool_key = (scheme, host, port)
# If the scheme, host, or port doesn't match existing open connections,
# open a new ConnectionPool.
- pool_key = (scheme, host, port or port_by_scheme.get(scheme, 80))
-
pool = self.pools.get(pool_key)
if pool:
return pool
return pool
+ def connection_from_url(self, url):
+ """
+ Similar to connectionpool.connection_from_url but doesn't pass any
+ additional keywords to the ConnectionPool constructor. Additional
+ keywords are taken from the PoolManager constructor.
+ """
+ scheme, host, port = get_host(url)
+
+ port = port or port_by_scheme.get(scheme, 80)
+
+ return self.connection_from_host(host, port=port, scheme=scheme)
+
def urlopen(self, method, url, **kw):
- "Same as HTTP(S)ConnectionPool.urlopen"
+ "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
conn = self.connection_from_url(url)
return conn.urlopen(method, url, **kw)