1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
10 from appengine_wrappers import urlfetch
11 from environment import GetAppVersion
12 from future import Future
16 _RETRY_DELAY_SECONDS = 30
19 def _MakeHeaders(username, password, access_token):
21 'User-Agent': 'Chromium docserver %s' % GetAppVersion(),
22 'Cache-Control': 'max-age=0',
24 if username is not None and password is not None:
25 headers['Authorization'] = 'Basic %s' % base64.b64encode(
26 '%s:%s' % (username, password))
27 if access_token is not None:
28 headers['Authorization'] = 'OAuth %s' % access_token
32 class AppEngineUrlFetcher(object):
33 """A wrapper around the App Engine urlfetch module that allows for easy
36 def __init__(self, base_path=None):
37 assert base_path is None or not base_path.endswith('/'), base_path
38 self._base_path = base_path
39 self._retries_left = _MAX_RETRIES
41 def Fetch(self, url, username=None, password=None, access_token=None):
42 """Fetches a file synchronously.
44 return urlfetch.fetch(self._FromBasePath(url),
46 headers=_MakeHeaders(username,
50 def FetchAsync(self, url, username=None, password=None, access_token=None):
51 """Fetches a file asynchronously, and returns a Future with the result.
53 def process_result(result):
54 if result.status_code == 429:
55 if self._retries_left == 0:
56 logging.error('Still throttled. Giving up.')
58 self._retries_left -= 1
59 logging.info('Throttled. Trying again in %s seconds.' %
61 time.sleep(_RETRY_DELAY_SECONDS)
62 return self.FetchAsync(url, username, password, access_token).Get()
65 rpc = urlfetch.create_rpc(deadline=20)
66 urlfetch.make_fetch_call(rpc,
67 self._FromBasePath(url),
68 headers=_MakeHeaders(username,
71 return Future(callback=lambda: process_result(rpc.get_result()))
73 def _FromBasePath(self, url):
74 assert not url.startswith('/'), url
75 if self._base_path is not None:
76 url = posixpath.join(self._base_path, url) if url else self._base_path