src/chrome/common/extensions/docs/server2/samples_data_source.py

   1 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import logging
   7 import posixpath
   8 import re
   9 import traceback
  10
  11 from extensions_paths import EXAMPLES
  12 import third_party.json_schema_compiler.json_comment_eater as json_comment_eater
  13 import url_constants
  14
  15
  16 _DEFAULT_ICON_PATH = 'images/sample-default-icon.png'
  17
  18
  19 class SamplesDataSource(object):
  20   '''Constructs a list of samples and their respective files and api calls.
  21   '''
  22   class Factory(object):
  23     '''A factory to create SamplesDataSource instances bound to individual
  24     Requests.
  25     '''
  26     def __init__(self,
  27                  host_file_system,
  28                  app_samples_file_system,
  29                  compiled_fs_factory,
  30                  ref_resolver_factory,
  31                  base_path):
  32       self._host_file_system = host_file_system
  33       self._app_samples_file_system = app_samples_file_system
  34       self._ref_resolver = ref_resolver_factory.Create()
  35       self._base_path = base_path
  36       self._extensions_cache = compiled_fs_factory.Create(
  37           host_file_system,
  38           self._MakeSamplesList,
  39           SamplesDataSource,
  40           category='extensions')
  41       self._extensions_text_cache = compiled_fs_factory.ForUnicode(
  42           host_file_system)
  43       self._apps_cache = compiled_fs_factory.Create(
  44           app_samples_file_system,
  45           lambda *args: self._MakeSamplesList(*args, is_apps=True),
  46           SamplesDataSource,
  47           category='apps')
  48       self._apps_text_cache = compiled_fs_factory.ForUnicode(
  49           app_samples_file_system)
  50
  51     def Create(self, request):
  52       '''Returns a new SamplesDataSource bound to |request|.
  53       '''
  54       return SamplesDataSource(self._extensions_cache,
  55                                self._apps_cache,
  56                                self._base_path,
  57                                request)
  58
  59     def _GetAPIItems(self, js_file):
  60       chrome_pattern = r'chrome[\w.]+'
  61       # Add API calls that appear normally, like "chrome.runtime.connect".
  62       calls = set(re.findall(chrome_pattern, js_file))
  63       # Add API calls that have been assigned into variables, like
  64       # "var storageArea = chrome.storage.sync; storageArea.get", which should
  65       # be expanded like "chrome.storage.sync.get".
  66       for match in re.finditer(r'var\s+(\w+)\s*=\s*(%s);' % chrome_pattern,
  67                                js_file):
  68         var_name, api_prefix = match.groups()
  69         for var_match in re.finditer(r'\b%s\.([\w.]+)\b' % re.escape(var_name),
  70                                      js_file):
  71           api_suffix, = var_match.groups()
  72           calls.add('%s.%s' % (api_prefix, api_suffix))
  73       return calls
  74
  75     def _GetDataFromManifest(self, path, text_cache, file_system):
  76       manifest = text_cache.GetFromFile(path + '/manifest.json').Get()
  77       try:
  78         manifest_json = json.loads(json_comment_eater.Nom(manifest))
  79       except ValueError as e:
  80         logging.error('Error parsing manifest.json for %s: %s' % (path, e))
  81         return None
  82       l10n_data = {
  83         'name': manifest_json.get('name', ''),
  84         'description': manifest_json.get('description', None),
  85         'icon': manifest_json.get('icons', {}).get('128', None),
  86         'default_locale': manifest_json.get('default_locale', None),
  87         'locales': {}
  88       }
  89       if not l10n_data['default_locale']:
  90         return l10n_data
  91       locales_path = path + '/_locales/'
  92       locales_dir = file_system.ReadSingle(locales_path).Get()
  93       if locales_dir:
  94         def load_locale_json(path):
  95           return (path, json.loads(text_cache.GetFromFile(path).Get()))
  96
  97         try:
  98           locales_json = [load_locale_json(locales_path + f + 'messages.json')
  99                           for f in locales_dir]
 100         except ValueError as e:
 101           logging.error('Error parsing locales files for %s: %s' % (path, e))
 102         else:
 103           for path, json_ in locales_json:
 104             l10n_data['locales'][path[len(locales_path):].split('/')[0]] = json_
 105       return l10n_data
 106
 107     def _MakeSamplesList(self, base_path, files, is_apps=False):
 108       file_system = (self._app_samples_file_system if is_apps else
 109                               self._host_file_system)
 110       text_cache = (self._apps_text_cache if is_apps else
 111           self._extensions_text_cache)
 112       samples_list = []
 113       for filename in sorted(files):
 114         if filename.rsplit('/')[-1] != 'manifest.json':
 115           continue
 116
 117         # This is a little hacky, but it makes a sample page.
 118         sample_path = filename.rsplit('/', 1)[-2]
 119         sample_files = [path for path in files
 120                         if path.startswith(sample_path + '/')]
 121         js_files = [path for path in sample_files if path.endswith('.js')]
 122         js_contents = [text_cache.GetFromFile(
 123             posixpath.join(base_path, js_file)).Get()
 124             for js_file in js_files]
 125         api_items = set()
 126         for js in js_contents:
 127           api_items.update(self._GetAPIItems(js))
 128
 129         api_calls = []
 130         for item in sorted(api_items):
 131           if len(item.split('.')) < 3:
 132             continue
 133           if item.endswith('.removeListener') or item.endswith('.hasListener'):
 134             continue
 135           if item.endswith('.addListener'):
 136             item = item[:-len('.addListener')]
 137           if item.startswith('chrome.'):
 138             item = item[len('chrome.'):]
 139           ref_data = self._ref_resolver.GetLink(item)
 140           # TODO(kalman): What about references like chrome.storage.sync.get?
 141           # That should link to either chrome.storage.sync or
 142           # chrome.storage.StorageArea.get (or probably both).
 143           # TODO(kalman): Filter out API-only references? This can happen when
 144           # the API namespace is assigned to a variable, but it's very hard to
 145           # to disambiguate.
 146           if ref_data is None:
 147             continue
 148           api_calls.append({
 149             'name': ref_data['text'],
 150             'link': ref_data['href']
 151           })
 152
 153         if is_apps:
 154           url = url_constants.GITHUB_BASE + '/' + sample_path
 155           icon_base = url_constants.RAW_GITHUB_BASE + '/' + sample_path
 156           download_url = url
 157         else:
 158           extension_sample_path = posixpath.join('examples', sample_path)
 159           url = extension_sample_path
 160           icon_base = extension_sample_path
 161           download_url = extension_sample_path + '.zip'
 162
 163         manifest_data = self._GetDataFromManifest(
 164             posixpath.join(base_path, sample_path),
 165             text_cache,
 166             file_system)
 167         if manifest_data['icon'] is None:
 168           icon_path = posixpath.join(
 169               self._base_path, 'static', _DEFAULT_ICON_PATH)
 170         else:
 171           icon_path = '%s/%s' % (icon_base, manifest_data['icon'])
 172         manifest_data.update({
 173           'icon': icon_path,
 174           'download_url': download_url,
 175           'url': url,
 176           'files': [f.replace(sample_path + '/', '') for f in sample_files],
 177           'api_calls': api_calls
 178         })
 179         samples_list.append(manifest_data)
 180
 181       return samples_list
 182
 183   def __init__(self,
 184                extensions_cache,
 185                apps_cache,
 186                base_path,
 187                request):
 188     self._extensions_cache = extensions_cache
 189     self._apps_cache = apps_cache
 190     self._base_path = base_path
 191     self._request = request
 192
 193   def _GetSampleId(self, sample_name):
 194     return sample_name.lower().replace(' ', '-')
 195
 196   def _GetAcceptedLanguages(self):
 197     accept_language = self._request.headers.get('Accept-Language', None)
 198     if accept_language is None:
 199       return []
 200     return [lang_with_q.split(';')[0].strip()
 201             for lang_with_q in accept_language.split(',')]
 202
 203   def FilterSamples(self, key, api_name):
 204     '''Fetches and filters the list of samples specified by |key|, returning
 205     only the samples that use the API |api_name|. |key| is either 'apps' or
 206     'extensions'.
 207     '''
 208     return [sample for sample in self.get(key) if any(
 209         call['name'].startswith(api_name + '.')
 210         for call in sample['api_calls'])]
 211
 212   def _CreateSamplesDict(self, key):
 213     if key == 'apps':
 214       samples_list = self._apps_cache.GetFromFileListing('').Get()
 215     else:
 216       samples_list = self._extensions_cache.GetFromFileListing(EXAMPLES).Get()
 217     return_list = []
 218     for dict_ in samples_list:
 219       name = dict_['name']
 220       description = dict_['description']
 221       if description is None:
 222         description = ''
 223       if name.startswith('__MSG_') or description.startswith('__MSG_'):
 224         try:
 225           # Copy the sample dict so we don't change the dict in the cache.
 226           sample_data = dict_.copy()
 227           name_key = name[len('__MSG_'):-len('__')]
 228           description_key = description[len('__MSG_'):-len('__')]
 229           locale = sample_data['default_locale']
 230           for lang in self._GetAcceptedLanguages():
 231             if lang in sample_data['locales']:
 232               locale = lang
 233               break
 234           locale_data = sample_data['locales'][locale]
 235           sample_data['name'] = locale_data[name_key]['message']
 236           sample_data['description'] = locale_data[description_key]['message']
 237           sample_data['id'] = self._GetSampleId(sample_data['name'])
 238         except Exception as e:
 239           logging.error(traceback.format_exc())
 240           # Revert the sample to the original dict.
 241           sample_data = dict_
 242         return_list.append(sample_data)
 243       else:
 244         dict_['id'] = self._GetSampleId(name)
 245         return_list.append(dict_)
 246     return return_list
 247
 248   def get(self, key):
 249     return {
 250       'apps': lambda: self._CreateSamplesDict('apps'),
 251       'extensions': lambda: self._CreateSamplesDict('extensions')
 252     }.get(key, lambda: {})()