src/chrome/common/extensions/docs/server2/github_file_system.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import logging
   7 import os
   8 from StringIO import StringIO
   9
  10 import appengine_blobstore as blobstore
  11 from appengine_url_fetcher import AppEngineUrlFetcher
  12 from appengine_wrappers import GetAppVersion, urlfetch
  13 from docs_server_utils import StringIdentity
  14 from file_system import FileSystem, StatInfo
  15 from future import Future
  16 from object_store_creator import ObjectStoreCreator
  17 import url_constants
  18 from zipfile import ZipFile, BadZipfile
  19
  20 ZIP_KEY = 'zipball'
  21 USERNAME = None
  22 PASSWORD = None
  23
  24 def _MakeBlobstoreKey(version):
  25   return ZIP_KEY + '.' + str(version)
  26
  27 class _AsyncFetchFutureZip(object):
  28   def __init__(self,
  29                fetcher,
  30                username,
  31                password,
  32                blobstore,
  33                key_to_set,
  34                key_to_delete=None):
  35     self._fetcher = fetcher
  36     self._fetch = fetcher.FetchAsync(ZIP_KEY,
  37                                      username=username,
  38                                      password=password)
  39     self._blobstore = blobstore
  40     self._key_to_set = key_to_set
  41     self._key_to_delete = key_to_delete
  42
  43   def Get(self):
  44     try:
  45       result = self._fetch.Get()
  46       # Check if Github authentication failed.
  47       if result.status_code == 401:
  48         logging.error('Github authentication failed for %s, falling back to '
  49                       'unauthenticated.' % USERNAME)
  50         blob = self._fetcher.Fetch(ZIP_KEY).content
  51       else:
  52         blob = result.content
  53     except urlfetch.DownloadError as e:
  54       logging.error('Bad github zip file: %s' % e)
  55       return None
  56     if self._key_to_delete is not None:
  57       self._blobstore.Delete(_MakeBlobstoreKey(self._key_to_delete),
  58                              blobstore.BLOBSTORE_GITHUB)
  59     try:
  60       return_zip = ZipFile(StringIO(blob))
  61     except BadZipfile as e:
  62       logging.error('Bad github zip file: %s' % e)
  63       return None
  64
  65     self._blobstore.Set(_MakeBlobstoreKey(self._key_to_set),
  66                         blob,
  67                         blobstore.BLOBSTORE_GITHUB)
  68     return return_zip
  69
  70 class GithubFileSystem(FileSystem):
  71   @staticmethod
  72   def CreateChromeAppsSamples(object_store_creator):
  73     return GithubFileSystem(
  74         '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS,
  75         blobstore.AppEngineBlobstore(),
  76         object_store_creator)
  77
  78   def __init__(self, url, blobstore, object_store_creator):
  79     # If we key the password store on the app version then the whole advantage
  80     # of having it in the first place is greatly lessened (likewise it should
  81     # always start populated).
  82     password_store = object_store_creator.Create(
  83         GithubFileSystem,
  84         app_version=None,
  85         category='password',
  86         start_empty=False)
  87     if USERNAME is None:
  88       password_data = password_store.GetMulti(('username', 'password')).Get()
  89       self._username, self._password = (password_data.get('username'),
  90                                         password_data.get('password'))
  91     else:
  92       password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
  93       self._username, self._password = (USERNAME, PASSWORD)
  94
  95     self._url = url
  96     self._fetcher = AppEngineUrlFetcher(url)
  97     self._blobstore = blobstore
  98     self._stat_object_store = object_store_creator.Create(GithubFileSystem)
  99     self._version = None
 100     self._GetZip(self.Stat(ZIP_KEY).version)
 101
 102   def _GetZip(self, version):
 103     blob = self._blobstore.Get(_MakeBlobstoreKey(version),
 104                                blobstore.BLOBSTORE_GITHUB)
 105     if blob is not None:
 106       try:
 107         self._zip_file = Future(value=ZipFile(StringIO(blob)))
 108       except BadZipfile as e:
 109         self._blobstore.Delete(_MakeBlobstoreKey(version),
 110                                blobstore.BLOBSTORE_GITHUB)
 111         logging.error('Bad github zip file: %s' % e)
 112         self._zip_file = Future(value=None)
 113     else:
 114       self._zip_file = Future(
 115           delegate=_AsyncFetchFutureZip(self._fetcher,
 116                                         self._username,
 117                                         self._password,
 118                                         self._blobstore,
 119                                         version,
 120                                         key_to_delete=self._version))
 121     self._version = version
 122
 123   def _ReadFile(self, path):
 124     try:
 125       zip_file = self._zip_file.Get()
 126     except Exception as e:
 127       logging.error('Github ReadFile error: %s' % e)
 128       return ''
 129     if zip_file is None:
 130       logging.error('Bad github zip file.')
 131       return ''
 132     prefix = zip_file.namelist()[0][:-1]
 133     return zip_file.read(prefix + path)
 134
 135   def _ListDir(self, path):
 136     try:
 137       zip_file = self._zip_file.Get()
 138     except Exception as e:
 139       logging.error('Github ListDir error: %s' % e)
 140       return []
 141     if zip_file is None:
 142       logging.error('Bad github zip file.')
 143       return []
 144     filenames = zip_file.namelist()
 145     # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
 146     filenames = [f[len(filenames[0]) - 1:] for f in filenames]
 147     # Remove the path of the directory we're listing from the filenames.
 148     filenames = [f[len(path):] for f in filenames
 149                  if f != path and f.startswith(path)]
 150     # Remove all files not directly in this directory.
 151     return [f for f in filenames if f[:-1].count('/') == 0]
 152
 153   def Read(self, paths, binary=False):
 154     version = self.Stat(ZIP_KEY).version
 155     if version != self._version:
 156       self._GetZip(version)
 157     result = {}
 158     for path in paths:
 159       if path.endswith('/'):
 160         result[path] = self._ListDir(path)
 161       else:
 162         result[path] = self._ReadFile(path)
 163     return Future(value=result)
 164
 165   def _DefaultStat(self, path):
 166     version = 0
 167     # TODO(kalman): we should replace all of this by wrapping the
 168     # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
 169     # CFS to be robust, and GFS is missing out.
 170     # For example: the following line is wrong, but it could be moot.
 171     self._stat_object_store.Set(path, version)
 172     return StatInfo(version)
 173
 174   def Stat(self, path):
 175     version = self._stat_object_store.Get(path).Get()
 176     if version is not None:
 177       return StatInfo(version)
 178     try:
 179       result = self._fetcher.Fetch('commits/HEAD',
 180                                    username=USERNAME,
 181                                    password=PASSWORD)
 182     except urlfetch.DownloadError as e:
 183       logging.warning('GithubFileSystem Stat: %s' % e)
 184       return self._DefaultStat(path)
 185
 186     # Check if Github authentication failed.
 187     if result.status_code == 401:
 188       logging.warning('Github authentication failed for %s, falling back to '
 189                       'unauthenticated.' % USERNAME)
 190       try:
 191         result = self._fetcher.Fetch('commits/HEAD')
 192       except urlfetch.DownloadError as e:
 193         logging.warning('GithubFileSystem Stat: %s' % e)
 194         return self._DefaultStat(path)
 195
 196     # Parse response JSON - but sometimes github gives us invalid JSON.
 197     try:
 198       version = json.loads(result.content)['commit']['tree']['sha']
 199       self._stat_object_store.Set(path, version)
 200       return StatInfo(version)
 201     except StandardError as e:
 202       logging.warning(
 203           ('%s: got invalid or unexpected JSON from github. Response status ' +
 204            'was %s, content %s') % (e, result.status_code, result.content))
 205       return self._DefaultStat(path)
 206
 207   def GetIdentity(self):
 208     return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))