import re
import socket
import ssl
-import sys
import threading
import time
import urllib
-import urllib2
import urlparse
from third_party import requests
from third_party.requests import adapters
from third_party.requests import structures
-from third_party.rietveld import upload
from utils import oauth
from utils import tools
-from utils import zip_package
-
-# Hack out upload logging.info()
-upload.logging = logging.getLogger('upload')
-# Mac pylint choke on this line.
-upload.logging.setLevel(logging.WARNING) # pylint: disable=E1103
# TODO(vadimsh): Remove this once we don't have to support python 2.6 anymore.
monkey_patch_httplib()
-# The name of the key to store the count of url attempts.
-COUNT_KEY = 'UrlOpenAttempt'
-
# Default maximum number of attempts to trying opening a url before aborting.
URL_OPEN_MAX_ATTEMPTS = 30
lambda x: json.dumps(x, sort_keys=True, separators=(',', ':')),
}
-# File to use to store all auth cookies.
-COOKIE_FILE = os.path.join(os.path.expanduser('~'), '.isolated_cookies')
# Google Storage URL regular expression.
GS_STORAGE_HOST_URL_RE = re.compile(r'https://.*\.storage\.googleapis\.com')
# Order is important: it's visible in commands --help output.
AUTH_METHODS = [
('oauth', oauth.OAuthConfig),
- ('cookie', None),
('bot', None),
('none', None),
]
-
# Global (for now) map: server URL (http://example.com) -> HttpService instance.
# Used by get_http_service to cache HttpService instances.
_http_services = {}
_http_services_lock = threading.Lock()
-# CookieJar reused by all services + lock that protects its instantiation.
-_cookie_jar = None
-_cookie_jar_lock = threading.Lock()
-
-# Path to cacert.pem bundle file reused by all services.
-_ca_certs = None
-_ca_certs_lock = threading.Lock()
-
# This lock ensures that user won't be confused with multiple concurrent
# login prompts.
_auth_lock = threading.Lock()
if verbose:
headers = None
body = None
- if isinstance(self.inner_exc, urllib2.HTTPError):
- headers = self.inner_exc.hdrs.items()
- body = self.inner_exc.read()
- elif isinstance(self.inner_exc, requests.HTTPError):
+ if isinstance(self.inner_exc, requests.HTTPError):
headers = self.inner_exc.response.headers.items()
body = self.inner_exc.response.content
if headers or body:
self.code = code
-def url_open(url, **kwargs):
+def url_open(url, **kwargs): # pylint: disable=W0621
"""Attempts to open the given url multiple times.
|data| can be either:
return None
+def url_read_json(url, **kwargs):
+ """Attempts to open the given url multiple times and read all data from it.
+
+ Accepts same arguments as url_open function.
+
+ Returns all data read or None if it was unable to connect or read the data.
+ """
+ urlhost, urlpath = split_server_request_url(url)
+ service = get_http_service(urlhost)
+ try:
+ return service.json_request(urlpath, **kwargs)
+ except TimeoutError:
+ return None
+
+
def url_retrieve(filepath, url, **kwargs):
"""Downloads an URL to a file. Returns True on success."""
response = url_open(url, **kwargs)
return urlhost, urlpath
-def get_http_service(urlhost, allow_cached=False, use_count_key=None):
+def get_http_service(urlhost, allow_cached=True):
"""Returns existing or creates new instance of HttpService that can send
requests to given base urlhost.
"""
def new_service():
return HttpService(
urlhost,
- engine=RequestsLibEngine(get_cacerts_bundle()),
- authenticator=create_authenticator(urlhost),
- use_count_key=use_count_key)
+ engine=RequestsLibEngine(),
+ authenticator=create_authenticator(urlhost))
# Ensure consistency in url naming.
urlhost = str(urlhost).lower().rstrip('/')
- # Do not use COUNT_KEY with Google Storage (since it breaks a signature).
- if use_count_key is None:
- use_count_key = not GS_STORAGE_HOST_URL_RE.match(urlhost)
-
if not allow_cached:
return new_service()
with _http_services_lock:
return service
-def get_cookie_jar():
- """Returns global CoookieJar object that stores cookies in the file."""
- global _cookie_jar
- with _cookie_jar_lock:
- if _cookie_jar is not None:
- return _cookie_jar
- jar = ThreadSafeCookieJar(COOKIE_FILE)
- jar.load()
- _cookie_jar = jar
- return jar
-
-
-def get_cacerts_bundle():
- """Returns path to a file with CA root certificates bundle."""
- global _ca_certs
- with _ca_certs_lock:
- if _ca_certs is not None and os.path.exists(_ca_certs):
- return _ca_certs
- _ca_certs = zip_package.extract_resource(requests, 'cacert.pem')
- return _ca_certs
-
-
def get_default_auth_config():
"""Returns auth configuration used by default if configure_auth is not called.
Possible authentication methods are:
'bot' - use HMAC authentication based on a secret key.
- 'cookie' - use cookie-based authentication.
- 'none' - do not use authentication.
'oauth' - use oauth-based authentication.
+ 'none' - do not use authentication.
Arguments:
method: what method to use.
# TODO(vadimsh): Implement it. Use IP whitelist (that doesn't require
# any authenticator instance) for now.
return None
- elif _auth_method == 'cookie':
- return CookieBasedAuthenticator(urlhost, get_cookie_jar())
- elif _auth_method == 'none':
- return None
elif _auth_method == 'oauth':
return OAuthAuthenticator(urlhost, _auth_method_config)
+ elif _auth_method == 'none':
+ return None
raise AssertionError('Invalid auth method: %s' % _auth_method)
- Thread safe.
"""
- def __init__(self, urlhost, engine, authenticator=None, use_count_key=True):
+ def __init__(self, urlhost, engine, authenticator=None):
self.urlhost = urlhost
self.engine = engine
self.authenticator = authenticator
- self.use_count_key = use_count_key
@staticmethod
def is_transient_http_error(code, retry_404, retry_50x):
failed (i.e. this function returns False).
'request' method always uses non-interactive login, so long-lived
- authentication tokens (cookie, OAuth2 refresh token, etc) have to be set up
+ authentication tokens (OAuth2 refresh token, etc) have to be set up
manually by developer (by calling 'auth.py login' perhaps) prior running
any swarming or isolate scripts.
"""
try:
# Prepare and send a new request.
- request = HttpRequest(method, resource_url, query_params, body,
+ request = HttpRequest(
+ method, resource_url, query_params, body,
headers, read_timeout, stream)
- self.prepare_request(request, attempt.attempt)
if self.authenticator:
self.authenticator.authorize(request)
response = self.engine.perform_request(request)
def json_request(
self,
- method,
urlpath,
- body=None,
+ method=None,
+ data=None,
max_attempts=URL_OPEN_MAX_ATTEMPTS,
timeout=URL_OPEN_TIMEOUT,
headers=None):
Arguments:
method: HTTP method to use ('GET', 'POST', ...).
urlpath: relative request path (e.g. '/auth/v1/...').
- body: object to serialize to JSON and sent in the request.
+ data: object to serialize to JSON and sent in the request.
max_attempts: how many times to retry 50x errors.
timeout: how long to wait for a response (including all retries).
headers: dict with additional request headers.
+ If |method| is given it can be 'GET', 'POST' or 'PUT' and it will be used
+ when performing the request. By default it's GET if |data| is None and POST
+ if |data| is not None.
+
Returns:
Deserialized JSON response on success, None on error or timeout.
"""
response = self.request(
urlpath,
- content_type=JSON_CONTENT_TYPE if body is not None else None,
- data=body,
+ content_type=JSON_CONTENT_TYPE if data is not None else None,
+ data=data,
headers=headers,
max_attempts=max_attempts,
- method=method,
retry_404=False,
retry_50x=True,
stream=False,
+ method=method,
timeout=timeout)
if not response:
return None
logging.error('Not a JSON response when calling %s: %s', urlpath, text)
return None
- def prepare_request(self, request, attempt): # pylint: disable=R0201
- """Modify HttpRequest before sending it by adding COUNT_KEY parameter."""
- # Add COUNT_KEY only on retries.
- if self.use_count_key and attempt:
- request.params += [(COUNT_KEY, attempt)]
-
class HttpRequest(object):
"""Request to HttpService."""
# Maximum number of internal connection retries in a connection pool.
CONNECTION_RETRIES = 0
- def __init__(self, ca_certs):
+ def __init__(self):
super(RequestsLibEngine, self).__init__()
self.session = requests.Session()
# Configure session.
self.session.trust_env = False
- self.session.verify = ca_certs
+ self.session.verify = tools.get_cacerts_bundle()
# Configure connection pools.
for protocol in ('https://', 'http://'):
self.session.mount(protocol, adapters.HTTPAdapter(
raise ConnectionError(e)
-# TODO(vadimsh): Remove once everything is using OAuth or HMAC-based auth.
-class CookieBasedAuthenticator(Authenticator):
- """Uses cookies (that AppEngine recognizes) to authenticate to |urlhost|."""
-
- def __init__(self, urlhost, cookie_jar):
- super(CookieBasedAuthenticator, self).__init__()
- self.urlhost = urlhost
- self.cookie_jar = cookie_jar
- self.email = None
- self.password = None
- self._keyring = None
- self._lock = threading.Lock()
-
- def authorize(self, request):
- # Copy all cookies from authenticator cookie jar to request cookie jar.
- with self._lock:
- with self.cookie_jar:
- for cookie in self.cookie_jar:
- request.cookies.set_cookie(cookie)
-
- def login(self, allow_user_interaction):
- # Cookie authentication is always interactive (it asks for user name).
- if not allow_user_interaction:
- print >> sys.stderr, 'Cookie authentication requires interactive login'
- return False
- # To be used from inside AuthServer.
- cookie_jar = self.cookie_jar
- # RPC server that uses AuthenticationSupport's cookie jar.
- class AuthServer(upload.AbstractRpcServer):
- def _GetOpener(self):
- # Authentication code needs to know about 302 response.
- # So make OpenerDirector without HTTPRedirectHandler.
- opener = urllib2.OpenerDirector()
- opener.add_handler(urllib2.ProxyHandler())
- opener.add_handler(urllib2.UnknownHandler())
- opener.add_handler(urllib2.HTTPHandler())
- opener.add_handler(urllib2.HTTPDefaultErrorHandler())
- opener.add_handler(urllib2.HTTPSHandler())
- opener.add_handler(urllib2.HTTPErrorProcessor())
- opener.add_handler(urllib2.HTTPCookieProcessor(cookie_jar))
- return opener
- def PerformAuthentication(self):
- self._Authenticate()
- return self.authenticated
- with self._lock:
- with cookie_jar:
- rpc_server = AuthServer(self.urlhost, self.get_credentials)
- return rpc_server.PerformAuthentication()
-
- def logout(self):
- domain = urlparse.urlparse(self.urlhost).netloc
- try:
- with self.cookie_jar:
- self.cookie_jar.clear(domain)
- except KeyError:
- pass
-
- def get_credentials(self):
- """Called during authentication process to get the credentials.
-
- May be called multiple times if authentication fails.
-
- Returns tuple (email, password).
- """
- if self.email and self.password:
- return (self.email, self.password)
- self._keyring = self._keyring or upload.KeyringCreds(self.urlhost,
- self.urlhost, self.email)
- return self._keyring.GetUserCredentials()
-
-
-# TODO(vadimsh): Remove once everything is using OAuth or HMAC-based auth.
-class ThreadSafeCookieJar(cookielib.MozillaCookieJar):
- """MozillaCookieJar with thread safe load and save."""
-
- def __enter__(self):
- """Context manager interface."""
- return self
-
- def __exit__(self, *_args):
- """Saves cookie jar when exiting the block."""
- self.save()
- return False
-
- def load(self, filename=None, ignore_discard=False, ignore_expires=False):
- """Loads cookies from the file if it exists."""
- filename = os.path.expanduser(filename or self.filename)
- with self._cookies_lock:
- if os.path.exists(filename):
- try:
- cookielib.MozillaCookieJar.load(
- self, filename, ignore_discard, ignore_expires)
- logging.debug('Loaded cookies from %s', filename)
- except (cookielib.LoadError, IOError):
- pass
- else:
- try:
- fd = os.open(filename, os.O_CREAT, 0600)
- os.close(fd)
- except OSError:
- logging.debug('Failed to create %s', filename)
- try:
- os.chmod(filename, 0600)
- except OSError:
- logging.debug('Failed to fix mode for %s', filename)
-
- def save(self, filename=None, ignore_discard=False, ignore_expires=False):
- """Saves cookies to the file, completely overwriting it."""
- logging.debug('Saving cookies to %s', filename or self.filename)
- with self._cookies_lock:
- try:
- cookielib.MozillaCookieJar.save(
- self, filename, ignore_discard, ignore_expires)
- except OSError:
- logging.error('Failed to save %s', filename)
-
-
class OAuthAuthenticator(Authenticator):
"""Uses OAuth Authorization header to authenticate requests."""
projects / platform / framework / web / crosswalk.git / blobdiff