• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import sys
6
7import schema_util
8from docs_server_utils import ToUnicode
9from file_system import FileNotFoundError
10from future import Gettable, Future
11from third_party.handlebar import Handlebar
12from third_party.json_schema_compiler import json_parse
13from third_party.json_schema_compiler.memoize import memoize
14
15
16_SINGLE_FILE_FUNCTIONS = set()
17
18
19def SingleFile(fn):
20  '''A decorator which can be optionally applied to the compilation function
21  passed to CompiledFileSystem.Create, indicating that the function only
22  needs access to the file which is given in the function's callback. When
23  this is the case some optimisations can be done.
24
25  Note that this decorator must be listed first in any list of decorators to
26  have any effect.
27  '''
28  _SINGLE_FILE_FUNCTIONS.add(fn)
29  return fn
30
31
32def Unicode(fn):
33  '''A decorator which can be optionally applied to the compilation function
34  passed to CompiledFileSystem.Create, indicating that the function processes
35  the file's data as Unicode text.
36  '''
37
38  # The arguments passed to fn can be (self, path, data) or (path, data). In
39  # either case the last argument is |data|, which should be converted to
40  # Unicode.
41  def convert_args(args):
42    args = list(args)
43    args[-1] = ToUnicode(args[-1])
44    return args
45
46  return lambda *args: fn(*convert_args(args))
47
48
49class _CacheEntry(object):
50  def __init__(self, cache_data, version):
51
52    self._cache_data = cache_data
53    self.version = version
54
55
56class CompiledFileSystem(object):
57  '''This class caches FileSystem data that has been processed.
58  '''
59
60  class Factory(object):
61    '''A class to build a CompiledFileSystem backed by |file_system|.
62    '''
63
64    def __init__(self, object_store_creator):
65      self._object_store_creator = object_store_creator
66
67    def Create(self, file_system, compilation_function, cls, category=None):
68      '''Creates a CompiledFileSystem view over |file_system| that populates
69      its cache by calling |compilation_function| with (path, data), where
70      |data| is the data that was fetched from |path| in |file_system|.
71
72      The namespace for the compiled file system is derived similar to
73      ObjectStoreCreator: from |cls| along with an optional |category|.
74      '''
75      assert isinstance(cls, type)
76      assert not cls.__name__[0].islower()  # guard against non-class types
77      full_name = [cls.__name__, file_system.GetIdentity()]
78      if category is not None:
79        full_name.append(category)
80      def create_object_store(my_category):
81        # The read caches can start populated (start_empty=False) because file
82        # updates are picked up by the stat - but only if the compilation
83        # function is affected by a single file. If the compilation function is
84        # affected by other files (e.g. compiling a list of APIs available to
85        # extensions may be affected by both a features file and the list of
86        # files in the API directory) then this optimisation won't work.
87        return self._object_store_creator.Create(
88            CompiledFileSystem,
89            category='/'.join(full_name + [my_category]),
90            start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS)
91      return CompiledFileSystem(file_system,
92                                compilation_function,
93                                create_object_store('file'),
94                                create_object_store('list'))
95
96    @memoize
97    def ForJson(self, file_system):
98      '''A CompiledFileSystem specifically for parsing JSON configuration data.
99      These are memoized over file systems tied to different branches.
100      '''
101      return self.Create(file_system,
102                         SingleFile(lambda _, data:
103                             json_parse.Parse(ToUnicode(data))),
104                         CompiledFileSystem,
105                         category='json')
106
107    @memoize
108    def ForApiSchema(self, file_system):
109      '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
110      data and formatting it so that it can be used by other classes, such
111      as Model and APISchemaGraph.
112      '''
113      return self.Create(file_system,
114                         SingleFile(Unicode(schema_util.ProcessSchema)),
115                         CompiledFileSystem,
116                         category='api-schema')
117
118    @memoize
119    def ForTemplates(self, file_system):
120      '''Creates a CompiledFileSystem for parsing templates.
121      '''
122      return self.Create(
123          file_system,
124          SingleFile(lambda path, text: Handlebar(ToUnicode(text), name=path)),
125          CompiledFileSystem)
126
127    @memoize
128    def ForUnicode(self, file_system):
129      '''Creates a CompiledFileSystem for Unicode text processing.
130      '''
131      return self.Create(
132        file_system,
133        SingleFile(lambda _, text: ToUnicode(text)),
134        CompiledFileSystem,
135        category='text')
136
137  def __init__(self,
138               file_system,
139               compilation_function,
140               file_object_store,
141               list_object_store):
142    self._file_system = file_system
143    self._compilation_function = compilation_function
144    self._file_object_store = file_object_store
145    self._list_object_store = list_object_store
146
147  def _RecursiveList(self, path):
148    '''Returns a Future containing the recursive directory listing of |path| as
149    a flat list of paths.
150    '''
151    def split_dirs_from_files(paths):
152      '''Returns a tuple (dirs, files) where |dirs| contains the directory
153      names in |paths| and |files| contains the files.
154      '''
155      result = [], []
156      for path in paths:
157        result[0 if path.endswith('/') else 1].append(path)
158      return result
159
160    def add_prefix(prefix, paths):
161      return [prefix + path for path in paths]
162
163    # Read in the initial list of files. Do this eagerly (i.e. not part of the
164    # asynchronous Future contract) because there's a greater chance to
165    # parallelise fetching with the second layer (can fetch multiple paths).
166    try:
167      first_layer_dirs, first_layer_files = split_dirs_from_files(
168          self._file_system.ReadSingle(path).Get())
169    except FileNotFoundError:
170      return Future(exc_info=sys.exc_info())
171
172    if not first_layer_dirs:
173      return Future(value=first_layer_files)
174
175    second_layer_listing = self._file_system.Read(
176        add_prefix(path, first_layer_dirs))
177
178    def resolve():
179      def get_from_future_listing(futures):
180        '''Recursively lists files from directory listing |futures|.
181        '''
182        dirs, files = [], []
183        for dir_name, listing in futures.Get().iteritems():
184          new_dirs, new_files = split_dirs_from_files(listing)
185          # |dirs| are paths for reading. Add the full prefix relative to
186          # |path| so that |file_system| can find the files.
187          dirs += add_prefix(dir_name, new_dirs)
188          # |files| are not for reading, they are for returning to the caller.
189          # This entire function set (i.e. GetFromFileListing) is defined to
190          # not include the fetched-path in the result, however, |dir_name|
191          # will be prefixed with |path|. Strip it.
192          assert dir_name.startswith(path)
193          files += add_prefix(dir_name[len(path):], new_files)
194        if dirs:
195          files += get_from_future_listing(self._file_system.Read(dirs))
196        return files
197
198      return first_layer_files + get_from_future_listing(second_layer_listing)
199
200    return Future(delegate=Gettable(resolve))
201
202  def GetFromFile(self, path):
203    '''Calls |compilation_function| on the contents of the file at |path|.  If
204    |binary| is True then the file will be read as binary - but this will only
205    apply for the first time the file is fetched; if already cached, |binary|
206    will be ignored.
207    '''
208    try:
209      version = self._file_system.Stat(path).version
210    except FileNotFoundError:
211      return Future(exc_info=sys.exc_info())
212
213    cache_entry = self._file_object_store.Get(path).Get()
214    if (cache_entry is not None) and (version == cache_entry.version):
215      return Future(value=cache_entry._cache_data)
216
217    future_files = self._file_system.ReadSingle(path)
218    def resolve():
219      cache_data = self._compilation_function(path, future_files.Get())
220      self._file_object_store.Set(path, _CacheEntry(cache_data, version))
221      return cache_data
222    return Future(delegate=Gettable(resolve))
223
224  def GetFromFileListing(self, path):
225    '''Calls |compilation_function| on the listing of the files at |path|.
226    Assumes that the path given is to a directory.
227    '''
228    if not path.endswith('/'):
229      path += '/'
230
231    try:
232      version = self._file_system.Stat(path).version
233    except FileNotFoundError:
234      return Future(exc_info=sys.exc_info())
235
236    cache_entry = self._list_object_store.Get(path).Get()
237    if (cache_entry is not None) and (version == cache_entry.version):
238      return Future(value=cache_entry._cache_data)
239
240    recursive_list_future = self._RecursiveList(path)
241    def resolve():
242      cache_data = self._compilation_function(path, recursive_list_future.Get())
243      self._list_object_store.Set(path, _CacheEntry(cache_data, version))
244      return cache_data
245    return Future(delegate=Gettable(resolve))
246
247  def GetFileVersion(self, path):
248    cache_entry = self._file_object_store.Get(path).Get()
249    if cache_entry is not None:
250      return cache_entry.version
251    return self._file_system.Stat(path).version
252
253  def GetFileListingVersion(self, path):
254    if not path.endswith('/'):
255      path += '/'
256    cache_entry = self._list_object_store.Get(path).Get()
257    if cache_entry is not None:
258      return cache_entry.version
259    return self._file_system.Stat(path).version
260