From: Kenneth Reitz Date: Sat, 15 Dec 2012 10:50:01 +0000 (-0500) Subject: WHOOOOOOOOOOOOOOOO X-Git-Tag: v1.0.0~63 X-Git-Url: http://review.tizen.org/git/?a=commitdiff_plain;h=92d57036f3805f1c6a7ff97758c0f70d17bd7191;p=services%2Fpython-requests.git WHOOOOOOOOOOOOOOOO --- diff --git a/requests/adapters.py b/requests/adapters.py index 002a8d4..e5001ee 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -117,8 +117,6 @@ class HTTPAdapter(BaseAdapter): - - def close(self): """Dispose of any internal state. diff --git a/requests/api.py b/requests/api.py index cb64b69..ac0a642 100644 --- a/requests/api.py +++ b/requests/api.py @@ -45,11 +45,8 @@ def request(method, url, **kwargs): session = sessions.session(config=kwargs.get('config', None)) adhoc_session = True - try: - return session.request(method=method, url=url, **kwargs) - finally: - if adhoc_session: - session.close() + + return session.request(method=method, url=url, **kwargs) def get(url, **kwargs): diff --git a/requests/models.py b/requests/models.py index 2400edd..342d3e4 100644 --- a/requests/models.py +++ b/requests/models.py @@ -17,7 +17,7 @@ from datetime import datetime from io import BytesIO from .hooks import dispatch_hook, HOOKS -zz +from .structures import CaseInsensitiveDict from .status_codes import codes from .auth import HTTPBasicAuth, HTTPProxyAuth @@ -172,7 +172,7 @@ class Request(object): self.params = params self.auth = auth self.cookies = cookies - self.timeout = timeout + # self.timeout = timeout # TODO: move to attached self.allow_redirects = allow_redirects self.proxies = proxies @@ -359,6 +359,495 @@ class PreparedRequest(RequestMixin): if cookie_header is not None: self.headers['Cookie'] = cookie_header + def send(self): + pass + # return + + # def prepare_allow_ + + + +class OldRequest(object): + """The :class:`Request ` object. It carries out all functionality + of Requests. Recommended interface is with the Requests functions. + """ + + def __init__(self, + url=None, + headers=dict(), + files=None, + method=None, + data=dict(), + params=dict(), + auth=None, + cookies=None, + timeout=None, + redirect=False, + allow_redirects=False, + proxies=None, + hooks=None, + config=None, + prefetch=True, + _poolmanager=None, + verify=None, + session=None, + cert=None): + + #: Dictionary of configurations for this request. + self.config = dict(config or []) + + #: Float describes the timeout of the request. + # (Use socket.setdefaulttimeout() as fallback) + self.timeout = timeout + + # URL + + #: Dictionary of HTTP Headers to attach to the :class:`Request `. + self.headers = dict(headers or []) + + #: Dictionary of files to multipart upload (``{filename: content}``). + self.files = None + + #: HTTP Method to use. + self.method = method + + #: Dictionary, bytes or file stream of request body data to attach to the + #: :class:`Request `. + self.data = None + + #: Dictionary of querystring data to attach to the + #: :class:`Request `. The dictionary values can be lists for representing + #: multivalued query parameters. + self.params = None + + #: True if :class:`Request ` is part of a redirect chain (disables history + #: and HTTPError storage). + self.redirect = redirect + + #: Set to True if full redirects are allowed (e.g. re-POST-ing of data at new ``Location``) + self.allow_redirects = allow_redirects + + # Dictionary mapping protocol to the URL of the proxy (e.g. {'http': 'foo.bar:3128'}) + self.proxies = dict(proxies or []) + + for proxy_type, uri_ref in list(self.proxies.items()): + if not uri_ref: + del self.proxies[proxy_type] + + # If no proxies are given, allow configuration by environment variables + # HTTP_PROXY and HTTPS_PROXY. + if not self.proxies and self.config.get('trust_env'): + self.proxies = get_environ_proxies(self.url) + + self.data = data + self.params = params + self.files = files + + #: :class:`Response ` instance, containing + #: content and metadata of HTTP Response, once :attr:`sent `. + self.response = Response() + + #: Authentication tuple or object to attach to :class:`Request `. + self.auth = auth + + # #: CookieJar to attach to :class:`Request `. + # if isinstance(cookies, cookielib.CookieJar): + # self.cookies = cookies + # else: + # self.cookies = cookiejar_from_dict(cookies) + + #: True if Request has been sent. + self.sent = False + + #: Event-handling hooks. + self.hooks = {} + + for event in HOOKS: + self.hooks[event] = [] + + hooks = hooks or {} + + for (k, v) in list(hooks.items()): + self.register_hook(event=k, hook=v) + + #: Session. + self.session = session + + #: SSL Verification. + self.verify = verify + + #: SSL Certificate + self.cert = cert + + #: Prefetch response content + self.prefetch = prefetch + + # if headers: + # headers = CaseInsensitiveDict(self.headers) + # else: + # headers = CaseInsensitiveDict() + + # Add configured base headers. + for (k, v) in list(self.config.get('base_headers', {}).items()): + if k not in headers: + headers[k] = v + + self.headers = headers + self._poolmanager = _poolmanager + + def __repr__(self): + return '' % (self.method) + + def _build_response(self, resp): + """Build internal :class:`Response ` object + from given response. + """ + + def build(resp): + + response = Response() + + # Pass settings over. + response.config = self.config + + if resp: + + # Fallback to None if there's no status_code, for whatever reason. + # response.status_code = getattr(resp, 'status', None) + + # Make headers case-insensitive. + # response.headers = CaseInsensitiveDict(getattr(resp, 'headers', {})) + + # Set encoding. + # response.encoding = get_encoding_from_headers(response.headers) + + # Add new cookies from the server. + extract_cookies_to_jar(self.cookies, self, resp) + + # Save cookies in Response. + response.cookies = self.cookies + + # Save cookies in Session. + for cookie in self.cookies: + self.session.cookies.set_cookie(cookie) + + # No exceptions were harmed in the making of this request. + response.error = getattr(resp, 'error', None) + + # Save original response for later. + response.raw = resp + if isinstance(self.full_url, bytes): + response.url = self.full_url.decode('utf-8') + else: + response.url = self.full_url + + return response + + history = [] + + r = build(resp) + # TODO: session level shit + if r.status_code in REDIRECT_STATI and not self.redirect: + + while (('location' in r.headers and r.status_code in REDIRECT_STATI) and + ((r.status_code is codes.see_other) or (self.allow_redirects))): + + r.content # Consume socket so it can be released + + if not len(history) < self.config.get('max_redirects'): + raise TooManyRedirects() + + # Release the connection back into the pool. + r.raw.release_conn() + + history.append(r) + + url = r.headers['location'] + data = self.data + files = self.files + + # Handle redirection without scheme (see: RFC 1808 Section 4) + if url.startswith('//'): + parsed_rurl = urlparse(r.url) + url = '%s:%s' % (parsed_rurl.scheme, url) + + # Facilitate non-RFC2616-compliant 'location' headers + # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') + if not urlparse(url).netloc: + url = urljoin(r.url, + # Compliant with RFC3986, we percent + # encode the url. + requote_uri(url)) + + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.3.4 + if r.status_code is codes.see_other: + method = 'GET' + data = None + files = None + else: + method = self.method + + # Do what the browsers do, despite standards... + if r.status_code in (codes.moved, codes.found) and self.method == 'POST': + method = 'GET' + data = None + files = None + + if (r.status_code == 303) and self.method != 'HEAD': + method = 'GET' + data = None + files = None + + # Remove the cookie headers that were sent. + headers = self.headers + try: + del headers['Cookie'] + except KeyError: + pass + + request = Request( + url=url, + headers=headers, + files=files, + method=method, + params=self.session.params, + auth=self.auth, + cookies=self.cookies, + redirect=True, + data=data, + config=self.config, + timeout=self.timeout, + _poolmanager=self._poolmanager, + proxies=self.proxies, + verify=self.verify, + session=self.session, + cert=self.cert, + prefetch=self.prefetch + ) + + request.send() + r = request.response + + r.history = history + + self.response = r + self.response.request = self + + + def register_hook(self, event, hook): + """Properly register a hook.""" + if isinstance(hook, collections.Callable): + self.hooks[event].append(hook) + elif hasattr(hook, '__iter__'): + self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + + def deregister_hook(self, event, hook): + """Deregister a previously registered hook. + Returns True if the hook existed, False if not. + """ + + try: + self.hooks[event].remove(hook) + return True + except ValueError: + return False + + def send(self, anyway=False, prefetch=None): + """Sends the request. Returns True if successful, False if not. + If there was an HTTPError during transmission, + self.response.status_code will contain the HTTPError code. + + Once a request is successfully sent, `sent` will equal True. + + :param anyway: If True, request will be sent, even if it has + already been sent. + + :param prefetch: If not None, will override the request's own setting + for prefetch. + """ + + # Build the URL + url = self.full_url + + # Pre-request hook. + r = dispatch_hook('pre_request', self.hooks, self) + self.__dict__.update(r.__dict__) + + # Logging + log.info('Sending %s: %s' % (self, url)) + + # Use .netrc auth if none was provided. + if not self.auth and self.config.get('trust_env'): + self.auth = get_netrc_auth(url) + + # if self.auth: + # if isinstance(self.auth, tuple) and len(self.auth) == 2: + # # special-case basic HTTP auth + # self.auth = HTTPBasicAuth(*self.auth) + + # # Allow auth to make its changes. + # r = self.auth(self) + + # # Update self to reflect the auth changes. + # self.__dict__.update(r.__dict__) + + # # Nottin' on you. + # body = None + # content_type = None + + # # Multi-part file uploads. + # if self.files: + # (body, content_type) = self._encode_files(self.files) + # else: + # if self.data: + + # body = self._encode_params(self.data) + # if isinstance(self.data, str) or isinstance(self.data, builtin_str) or hasattr(self.data, 'read'): + # content_type = None + # else: + # content_type = 'application/x-www-form-urlencoded' + + # self.headers['Content-Length'] = '0' + # if hasattr(body, 'seek') and hasattr(body, 'tell'): + # body.seek(0, 2) + # self.headers['Content-Length'] = str(body.tell()) + # body.seek(0, 0) + # elif body is not None: + # self.headers['Content-Length'] = str(len(body)) + + # # Add content-type if it wasn't explicitly provided. + # if (content_type) and (not 'content-type' in self.headers): + # self.headers['Content-Type'] = content_type + + _p = urlparse(url) + no_proxy = filter(lambda x: x.strip(), self.proxies.get('no', '').split(',')) + proxy = self.proxies.get(_p.scheme) + + if proxy and not any(map(_p.hostname.endswith, no_proxy)): + conn = poolmanager.ProxyManager(self.get_connection_for_url(proxy)) + _proxy = urlparse(proxy) + if '@' in _proxy.netloc: + auth, url = _proxy.netloc.split('@', 1) + self.proxy_auth = HTTPProxyAuth(*auth.split(':', 1)) + r = self.proxy_auth(self) + self.__dict__.update(r.__dict__) + else: + conn = self.get_connection_for_url(url) + + if not self.config.get('keep_alive'): + self.headers['Connection'] = 'close' + + # if url.startswith('https') and self.verify: + + # cert_loc = None + + # # Allow self-specified cert location. + # if self.verify is not True: + # cert_loc = self.verify + + # # Look for configuration. + # if not cert_loc and self.config.get('trust_env'): + # cert_loc = os.environ.get('REQUESTS_CA_BUNDLE') + + # # Curl compatibility. + # if not cert_loc and self.config.get('trust_env'): + # cert_loc = os.environ.get('CURL_CA_BUNDLE') + + # if not cert_loc: + # cert_loc = DEFAULT_CA_BUNDLE_PATH + + # if not cert_loc: + # raise Exception("Could not find a suitable SSL CA certificate bundle.") + + # conn.cert_reqs = 'CERT_REQUIRED' + # conn.ca_certs = cert_loc + # else: + # conn.cert_reqs = 'CERT_NONE' + # conn.ca_certs = None + + # if self.cert: + # if len(self.cert) == 2: + # conn.cert_file = self.cert[0] + # conn.key_file = self.cert[1] + # else: + # conn.cert_file = self.cert + + if not self.sent or anyway: + + # Skip if 'cookie' header is explicitly set. + # if 'cookie' not in self.headers: + # cookie_header = get_cookie_header(self.cookies, self) + # if cookie_header is not None: + # self.headers['Cookie'] = cookie_header + + # Pre-send hook. + r = dispatch_hook('pre_send', self.hooks, self) + self.__dict__.update(r.__dict__) + + # catch urllib3 exceptions and throw Requests exceptions + # try: + # # Send the request. + # r = conn.urlopen( + # method=self.method, + # url=self.path_url, + # body=body, + # headers=self.headers, + # redirect=False, + # assert_same_host=False, + # preload_content=False, + # decode_content=False, + # retries=self.config.get('max_retries', 0), + # timeout=self.timeout, + # ) + # self.sent = True + + # except socket.error as sockerr: + # raise ConnectionError(sockerr) + + # except MaxRetryError as e: + # raise ConnectionError(e) + + # except (_SSLError, _HTTPError) as e: + # if isinstance(e, _SSLError): + # raise SSLError(e) + # elif isinstance(e, TimeoutError): + # raise Timeout(e) + # else: + # raise Timeout('Request timed out.') + + # build_response can throw TooManyRedirects + self._build_response(r) + + # Response manipulation hook. + self.response = dispatch_hook('response', self.hooks, self.response) + + # Post-request hook. + r = dispatch_hook('post_request', self.hooks, self) + self.__dict__.update(r.__dict__) + + # If prefetch is True, mark content as consumed. + if prefetch is None: + prefetch = self.prefetch + if prefetch: + # Save the response. + self.response.content + + return self.sent + + def get_connection_for_url(self, url): + # Check to see if keep_alive is allowed. + try: + if self.config.get('keep_alive'): + conn = self._poolmanager.connection_from_url(url) + else: + conn = connectionpool.connection_from_url(url) + return conn + except LocationParseError as e: + raise InvalidURL(e) + + def prepare(self): + return deepcopy(self) + class BaseResponse(object): """The basic Respone""" @@ -574,6 +1063,7 @@ class Response(BaseResponse): header = self.headers['link'] + # l = MultiDict() l = {} if header: diff --git a/requests/sessions.py b/requests/sessions.py index a8e75f8..3e91a00 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -19,6 +19,8 @@ from .utils import header_expand, from_key_val_list from .packages.urllib3.poolmanager import PoolManager +from adapters import HTTPAdapter + def merge_kwargs(local_kwarg, default_kwarg): """Merges kwarg dictionaries. @@ -114,24 +116,12 @@ class Session(object): for (k, v) in list(defaults.items()): self.config.setdefault(k, deepcopy(v)) - self.init_poolmanager() - # Set up a CookieJar to be used by default if isinstance(cookies, cookielib.CookieJar): self.cookies = cookies else: self.cookies = cookiejar_from_dict(cookies) - def init_poolmanager(self): - self.poolmanager = PoolManager( - num_pools=self.config.get('pool_connections'), - maxsize=self.config.get('pool_maxsize'), - strict=not self.config.get('support_http0.9') - ) - - def __repr__(self): - return '' % (id(self)) - def __enter__(self): return self @@ -139,12 +129,7 @@ class Session(object): self.close() def close(self): - """Dispose of any internal state. - - Currently, this just closes the PoolManager, which closes pooled - connections. - """ - self.poolmanager.clear() + pass def request(self, method, url, params=None, @@ -163,6 +148,44 @@ class Session(object): verify=None, cert=None): + req = Request() + req.method = method + req.url = url + req.headers = headers + req.files = files + req.data = data + req.params = params + req.auth = auth + req.cookies = cookies + # TODO: move to attached + req.allow_redirects = allow_redirects + req.proxies = proxies + req.hooks = hooks + + prep = req.prepare() + + return self.send(prep) + + + + + def old_request(self, method, url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=True, + proxies=None, + hooks=None, + return_response=True, + config=None, + prefetch=None, + verify=None, + cert=None): + """Constructs and sends a :class:`Request `. Returns :class:`Response ` object. @@ -268,13 +291,6 @@ class Session(object): # Send the HTTP Request. return self._send_request(r, **args) - def _send_request(self, r, **kwargs): - # Send the request. - r.send(prefetch=kwargs.get("prefetch")) - - # Return the response. - return r.response - def get(self, url, **kwargs): """Sends a GET request. Returns :class:`Response` object. @@ -346,7 +362,9 @@ class Session(object): def send(self, request): """Send a given PreparedRequest.""" - return request.send() + adapter = HTTPAdapter() + r = adapter.send(request) + return r def __getstate__(self): return dict((attr, getattr(self, attr, None)) for attr in self.__attrs__)