From d3acb783b95623d0378db02bd9832fb11850204b Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Fri, 15 Jun 2012 14:31:19 -0400 Subject: [PATCH] urllib3 update fixes #664 --- requests/packages/urllib3/connectionpool.py | 5 ++- requests/packages/urllib3/filepost.py | 13 +++--- .../packages/mimetools_choose_boundary/__init__.py | 47 ---------------------- requests/packages/urllib3/util.py | 37 +++++++++++++++-- 4 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py diff --git a/requests/packages/urllib3/connectionpool.py b/requests/packages/urllib3/connectionpool.py index 336aa77..f233341 100644 --- a/requests/packages/urllib3/connectionpool.py +++ b/requests/packages/urllib3/connectionpool.py @@ -258,7 +258,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if sock: sock.settimeout(timeout) - httplib_response = conn.getresponse() + try: # Python 2.7+, use buffering of HTTP responses + httplib_response = conn.getresponse(buffering=True) + except TypeError: # Python 2.6 and older + httplib_response = conn.getresponse() # AppEngine doesn't have a version attr. http_version = getattr(conn, '_http_vsn_str', 'HTTP/?'), diff --git a/requests/packages/urllib3/filepost.py b/requests/packages/urllib3/filepost.py index 344a103..e679b93 100644 --- a/requests/packages/urllib3/filepost.py +++ b/requests/packages/urllib3/filepost.py @@ -7,11 +7,7 @@ import codecs import mimetypes -try: - from mimetools import choose_boundary -except ImportError: - from .packages.mimetools_choose_boundary import choose_boundary - +from uuid import uuid4 from io import BytesIO from .packages import six @@ -20,6 +16,13 @@ from .packages.six import b writer = codecs.lookup('utf-8')[3] +def choose_boundary(): + """ + Our embarassingly-simple replacement for mimetools.choose_boundary. + """ + return uuid4().hex + + def get_content_type(filename): return mimetypes.guess_type(filename)[0] or 'application/octet-stream' diff --git a/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py b/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py deleted file mode 100644 index a0109ab..0000000 --- a/requests/packages/urllib3/packages/mimetools_choose_boundary/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -"""The function mimetools.choose_boundary() from Python 2.7, which seems to -have disappeared in Python 3 (although email.generator._make_boundary() might -work as a replacement?). - -Tweaked to use lock from threading rather than thread. -""" -import os -from threading import Lock -_counter_lock = Lock() - -_counter = 0 -def _get_next_counter(): - global _counter - with _counter_lock: - _counter += 1 - return _counter - -_prefix = None - -def choose_boundary(): - """Return a string usable as a multipart boundary. - - The string chosen is unique within a single program run, and - incorporates the user id (if available), process id (if available), - and current time. So it's very unlikely the returned string appears - in message text, but there's no guarantee. - - The boundary contains dots so you have to quote it in the header.""" - - global _prefix - import time - if _prefix is None: - import socket - try: - hostid = socket.gethostbyname(socket.gethostname()) - except socket.gaierror: - hostid = '127.0.0.1' - try: - uid = repr(os.getuid()) - except AttributeError: - uid = '1' - try: - pid = repr(os.getpid()) - except AttributeError: - pid = '1' - _prefix = hostid + '.' + uid + '.' + pid - return "%s.%.3f.%d" % (_prefix, time.time(), _get_next_counter()) diff --git a/requests/packages/urllib3/util.py b/requests/packages/urllib3/util.py index 2684a2f..486e8fa 100644 --- a/requests/packages/urllib3/util.py +++ b/requests/packages/urllib3/util.py @@ -72,6 +72,32 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, return headers +def split_first(s, delims): + """ + Given a string and an iterable of delimiters, split on the first found + delimiter. Return two split parts. + + If not found, then the first part is the full input string. + + Scales linearly with number of delims. Not ideal for large number of delims. + """ + min_idx = None + for d in delims: + idx = s.find(d) + if idx < 0: + continue + + if not min_idx: + min_idx = idx + else: + min_idx = min(idx, min_idx) + + if min_idx < 0: + return s, '' + + return s[:min_idx], s[min_idx+1:] + + def get_host(url): """ Given a url, return its scheme, host and port (None if it's not there). @@ -91,8 +117,11 @@ def get_host(url): if '://' in url: scheme, url = url.split('://', 1) - if '/' in url: - url, _path = url.split('/', 1) + + # Find the earliest Authority Terminator + # http://tools.ietf.org/html/rfc3986#section-3.2 + url, _path = split_first(url, ['/', '?', '#']) + if '@' in url: _auth, url = url.split('@', 1) if ':' in url: @@ -118,11 +147,11 @@ def is_connection_dropped(conn): let the platform handle connection recycling transparently for us. """ sock = getattr(conn, 'sock', False) - if not sock: #Platform-specific: AppEngine + if not sock: # Platform-specific: AppEngine return False if not poll: # Platform-specific - if not select: #Platform-specific: AppEngine + if not select: # Platform-specific: AppEngine return False return select([sock], [], [], 0.0)[0] -- 2.7.4