python3 version of urllib3
authorKenneth Reitz <me@kennethreitz.com>
Sun, 22 Jan 2012 20:06:53 +0000 (15:06 -0500)
committerKenneth Reitz <me@kennethreitz.com>
Sun, 22 Jan 2012 20:06:58 +0000 (15:06 -0500)
requests/packages/urllib3/__init__.py
requests/packages/urllib3/_collections.py
requests/packages/urllib3/connectionpool.py
requests/packages/urllib3/contrib/ntlmpool.py
requests/packages/urllib3/exceptions.py
requests/packages/urllib3/filepost.py
requests/packages/urllib3/poolmanager.py
requests/packages/urllib3/request.py
requests/packages/urllib3/response.py
requests/packages/urllib3/six.py [new file with mode: 0644]

index 20b1fb4..5f70c56 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/__init__.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -10,7 +10,7 @@ urllib3 - Thread-safe connection pooling and re-using.
 
 __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)'
 __license__ = 'MIT'
-__version__ = '1.0.2'
+__version__ = '1.1'
 
 
 from .connectionpool import (
index 00b2cd5..3cef081 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/_collections.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
index 17f2f84..1d1f9a0 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/connectionpool.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -7,9 +7,16 @@
 import logging
 import socket
 
+try:   # Python 3
+    from http.client import HTTPConnection, HTTPSConnection, HTTPException
+except ImportError:
+    from httplib import HTTPConnection, HTTPSConnection, HTTPException
 
-from httplib import HTTPConnection, HTTPSConnection, HTTPException
-from Queue import Queue, Empty, Full
+try:   # Python 3
+    from queue import Queue, Empty, Full
+except ImportError:
+    from Queue import Queue, Empty, Full
+    
 from select import select
 from socket import error as SocketError, timeout as SocketTimeout
 
@@ -33,6 +40,9 @@ from .exceptions import (
     EmptyPoolError,
 )
 
+from . import six
+xrange = six.moves.xrange
+
 
 log = logging.getLogger(__name__)
 
@@ -351,27 +361,29 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
             #     ``response.release_conn()`` is called (implicitly by
             #     ``response.read()``)
 
-        except (Empty), e:
+        except Empty as e:
             # Timed out by queue
             raise TimeoutError("Request timed out. (pool_timeout=%s)" %
                                pool_timeout)
 
-        except (SocketTimeout), e:
+        except SocketTimeout as e:
             # Timed out by socket
             raise TimeoutError("Request timed out. (timeout=%s)" %
                                timeout)
 
-        except (BaseSSLError), e:
+        except BaseSSLError as e:
             # SSL certificate error
             raise SSLError(e)
 
-        except (CertificateError), e:
+        except CertificateError as e:
             # Name mismatch
             raise SSLError(e)
 
-        except (HTTPException, SocketError), e:
+        except (HTTPException, SocketError) as e:
             # Connection broken, discard. It will be replaced next _get_conn().
             conn = None
+            # This is necessary so we can access e below
+            err = e
 
         finally:
             if conn and release_conn:
@@ -380,7 +392,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
 
         if not conn:
             log.warn("Retrying (%d attempts remain) after connection "
-                     "broken by '%r': %s" % (retries, e, url))
+                     "broken by '%r': %s" % (retries, err, url))
             return self.urlopen(method, url, body, headers, retries - 1,
                                 redirect, assert_same_host)  # Try again
 
index c5f010e..bb41fd1 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/contrib/ntlmpool.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -10,7 +10,10 @@ NTLM authenticating pool, contributed by erikcederstran
 Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10
 """
 
-import httplib
+try:
+    from http.client import HTTPSConnection
+except ImportError:
+    from httplib import HTTPSConnection
 from logging import getLogger
 from ntlm import ntlm
 
@@ -53,7 +56,7 @@ class NTLMConnectionPool(HTTPSConnectionPool):
         req_header = 'Authorization'
         resp_header = 'www-authenticate'
 
-        conn = httplib.HTTPSConnection(host=self.host, port=self.port)
+        conn = HTTPSConnection(host=self.host, port=self.port)
 
         # Send negotiation message
         headers[req_header] = (
index 47937f7..e556703 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/exceptions.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
index 2ffea8b..21b0e59 100644 (file)
@@ -1,18 +1,22 @@
 # urllib3/filepost.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
 import codecs
-import mimetools
 import mimetypes
 
 try:
-    from cStringIO import StringIO
+    from mimetools import choose_boundary
 except ImportError:
-    from StringIO import StringIO # pylint: disable-msg=W0404
+    # I don't like using an undocumented function, but I don't yet know what it does
+    from email.generator import _make_boundary as choose_boundary
 
+from io import BytesIO
+
+from . import six
+from .six import b
 
 writer = codecs.lookup('utf-8')[3]
 
@@ -35,37 +39,37 @@ def encode_multipart_formdata(fields, boundary=None):
         If not specified, then a random boundary will be generated using
         :func:`mimetools.choose_boundary`.
     """
-    body = StringIO()
+    body = BytesIO()
     if boundary is None:
-        boundary = mimetools.choose_boundary()
+        boundary = choose_boundary()
 
-    for fieldname, value in fields.iteritems():
-        body.write('--%s\r\n' % (boundary))
+    for fieldname, value in six.iteritems(fields):
+        body.write(b('--%s\r\n' % (boundary)))
 
         if isinstance(value, tuple):
             filename, data = value
             writer(body).write('Content-Disposition: form-data; name="%s"; '
                                'filename="%s"\r\n' % (fieldname, filename))
-            body.write('Content-Type: %s\r\n\r\n' %
-                       (get_content_type(filename)))
+            body.write(b('Content-Type: %s\r\n\r\n' %
+                       (get_content_type(filename))))
         else:
             data = value
             writer(body).write('Content-Disposition: form-data; name="%s"\r\n'
                                % (fieldname))
-            body.write('Content-Type: text/plain\r\n\r\n')
+            body.write(b'Content-Type: text/plain\r\n\r\n')
 
         if isinstance(data, int):
             data = str(data)  # Backwards compatibility
 
-        if isinstance(data, unicode):
+        if isinstance(data, six.text_type):
             writer(body).write(data)
         else:
             body.write(data)
 
-        body.write('\r\n')
+        body.write(b'\r\n')
 
-    body.write('--%s--\r\n' % (boundary))
+    body.write(b('--%s--\r\n' % (boundary)))
 
-    content_type = 'multipart/form-data; boundary=%s' % boundary
+    content_type = b('multipart/form-data; boundary=%s' % boundary)
 
     return body.getvalue(), content_type
index 482ee4a..37a5349 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/poolmanager.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -108,7 +108,7 @@ class PoolManager(RequestMethods):
         try:
             return conn.urlopen(method, url, **kw)
 
-        except HostChangedError, e:
+        except HostChangedError as e:
             kw['retries'] = e.retries # Persist retries countdown
             return self.urlopen(method, e.new_url, **kw)
 
index a7e0b5d..5ea26a0 100644 (file)
@@ -1,11 +1,13 @@
 # urllib3/request.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
 
-
-from urllib import urlencode
+try:
+    from urllib.parse import urlencode
+except ImportError:
+    from urllib import urlencode
 
 from .filepost import encode_multipart_formdata
 
index ee2ff66..5038974 100644 (file)
@@ -1,5 +1,5 @@
 # urllib3/response.py
-# Copyright 2008-2011 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
+# Copyright 2008-2012 Andrey Petrov and contributors (see CONTRIBUTORS.txt)
 #
 # This module is part of urllib3 and is released under
 # the MIT License: http://www.opensource.org/licenses/mit-license.php
@@ -8,11 +8,7 @@ import gzip
 import logging
 import zlib
 
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO # pylint: disable-msg=W0404
+from io import BytesIO
 
 
 from .exceptions import HTTPError
@@ -22,7 +18,7 @@ log = logging.getLogger(__name__)
 
 
 def decode_gzip(data):
-    gzipper = gzip.GzipFile(fileobj=StringIO(data))
+    gzipper = gzip.GzipFile(fileobj=BytesIO(data))
     return gzipper.read()
 
 
@@ -178,12 +174,15 @@ class HTTPResponse(object):
         with ``original_response=r``.
         """
 
+        # HTTPResponse objects in Python 3 don't have a .strict attribute
+        strict = getattr(r, 'strict', 0)
         return ResponseCls(body=r,
-                           headers=dict(r.getheaders()),
+                           # In Python 3, the header keys are returned capitalised
+                           headers=dict((k.lower(), v) for k,v in r.getheaders()),
                            status=r.status,
                            version=r.version,
                            reason=r.reason,
-                           strict=r.strict,
+                           strict=strict,
                            original_response=r,
                            **response_kw)
 
diff --git a/requests/packages/urllib3/six.py b/requests/packages/urllib3/six.py
new file mode 100644 (file)
index 0000000..a64f6fb
--- /dev/null
@@ -0,0 +1,372 @@
+"""Utilities for writing code that runs on Python 2 and 3"""
+
+#Copyright (c) 2010-2011 Benjamin Peterson
+
+#Permission is hereby granted, free of charge, to any person obtaining a copy of
+#this software and associated documentation files (the "Software"), to deal in
+#the Software without restriction, including without limitation the rights to
+#use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+#the Software, and to permit persons to whom the Software is furnished to do so,
+#subject to the following conditions:
+
+#The above copyright notice and this permission notice shall be included in all
+#copies or substantial portions of the Software.
+
+#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+#FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+#COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+#IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+#CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.1.0"
+
+
+# True if we are running on Python 3.
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+    class X(object):
+        def __len__(self):
+            return 1 << 31
+    try:
+        len(X())
+    except OverflowError:
+        # 32-bit
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # 64-bit
+        MAXSIZE = int((1 << 63) - 1)
+    del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)
+        # This is a bit ugly, but it avoids running this again.
+        delattr(tp, self.name)
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+
+class _MovedItems(types.ModuleType):
+    """Lazy loading of moved objects"""
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("winreg", "_winreg"),
+]
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+del attr
+
+moves = sys.modules["six.moves"] = _MovedItems("moves")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+
+    _iterkeys = "keys"
+    _itervalues = "values"
+    _iteritems = "items"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+
+    _iterkeys = "iterkeys"
+    _itervalues = "itervalues"
+    _iteritems = "iteritems"
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+
+    advance_iterator = next
+
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+
+    def advance_iterator(it):
+        return it.next()
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+
+
+def iterkeys(d):
+    """Return an iterator over the keys of a dictionary."""
+    return getattr(d, _iterkeys)()
+
+def itervalues(d):
+    """Return an iterator over the values of a dictionary."""
+    return getattr(d, _itervalues)()
+
+def iteritems(d):
+    """Return an iterator over the (key, value) pairs of a dictionary."""
+    return getattr(d, _iteritems)()
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+    def u(s):
+        return s
+    if sys.version_info[1] <= 1:
+        def int2byte(i):
+            return bytes((i,))
+    else:
+        # This is about 2x faster than the implementation above on 3.2+
+        int2byte = operator.methodcaller("to_bytes", 1, "big")
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+else:
+    def b(s):
+        return s
+    def u(s):
+        return unicode(s, "unicode_escape")
+    int2byte = chr
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+if PY3:
+    import builtins
+    exec_ = getattr(builtins, "exec")
+
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+
+    print_ = getattr(builtins, "print")
+    del builtins
+
+else:
+    def exec_(code, globs=None, locs=None):
+        """Execute code in a namespace."""
+        if globs is None:
+            frame = sys._getframe(1)
+            globs = frame.f_globals
+            if locs is None:
+                locs = frame.f_locals
+            del frame
+        elif locs is None:
+            locs = globs
+        exec("""exec code in globs, locs""")
+
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+    def print_(*args, **kwargs):
+        """The new-style print function."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+
+_add_doc(reraise, """Reraise an exception.""")
+
+
+def with_metaclass(meta, base=object):
+    """Create a base class with a metaclass."""
+    return meta("NewBase", (base,), {})