• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import sys
6
7import schema_util
8from docs_server_utils import ToUnicode
9from file_system import FileNotFoundError
10from future import Future
11from path_util import AssertIsDirectory, AssertIsFile
12from third_party.handlebar import Handlebar
13from third_party.json_schema_compiler import json_parse
14from third_party.json_schema_compiler.memoize import memoize
15
16
17_SINGLE_FILE_FUNCTIONS = set()
18
19
20def SingleFile(fn):
21  '''A decorator which can be optionally applied to the compilation function
22  passed to CompiledFileSystem.Create, indicating that the function only
23  needs access to the file which is given in the function's callback. When
24  this is the case some optimisations can be done.
25
26  Note that this decorator must be listed first in any list of decorators to
27  have any effect.
28  '''
29  _SINGLE_FILE_FUNCTIONS.add(fn)
30  return fn
31
32
33def Unicode(fn):
34  '''A decorator which can be optionally applied to the compilation function
35  passed to CompiledFileSystem.Create, indicating that the function processes
36  the file's data as Unicode text.
37  '''
38
39  # The arguments passed to fn can be (self, path, data) or (path, data). In
40  # either case the last argument is |data|, which should be converted to
41  # Unicode.
42  def convert_args(args):
43    args = list(args)
44    args[-1] = ToUnicode(args[-1])
45    return args
46
47  return lambda *args: fn(*convert_args(args))
48
49
50class _CacheEntry(object):
51  def __init__(self, cache_data, version):
52
53    self._cache_data = cache_data
54    self.version = version
55
56
57class CompiledFileSystem(object):
58  '''This class caches FileSystem data that has been processed.
59  '''
60
61  class Factory(object):
62    '''A class to build a CompiledFileSystem backed by |file_system|.
63    '''
64
65    def __init__(self, object_store_creator):
66      self._object_store_creator = object_store_creator
67
68    def Create(self, file_system, compilation_function, cls, category=None):
69      '''Creates a CompiledFileSystem view over |file_system| that populates
70      its cache by calling |compilation_function| with (path, data), where
71      |data| is the data that was fetched from |path| in |file_system|.
72
73      The namespace for the compiled file system is derived similar to
74      ObjectStoreCreator: from |cls| along with an optional |category|.
75      '''
76      assert isinstance(cls, type)
77      assert not cls.__name__[0].islower()  # guard against non-class types
78      full_name = [cls.__name__, file_system.GetIdentity()]
79      if category is not None:
80        full_name.append(category)
81      def create_object_store(my_category):
82        # The read caches can start populated (start_empty=False) because file
83        # updates are picked up by the stat - but only if the compilation
84        # function is affected by a single file. If the compilation function is
85        # affected by other files (e.g. compiling a list of APIs available to
86        # extensions may be affected by both a features file and the list of
87        # files in the API directory) then this optimisation won't work.
88        return self._object_store_creator.Create(
89            CompiledFileSystem,
90            category='/'.join(full_name + [my_category]),
91            start_empty=compilation_function not in _SINGLE_FILE_FUNCTIONS)
92      return CompiledFileSystem(file_system,
93                                compilation_function,
94                                create_object_store('file'),
95                                create_object_store('list'))
96
97    @memoize
98    def ForJson(self, file_system):
99      '''A CompiledFileSystem specifically for parsing JSON configuration data.
100      These are memoized over file systems tied to different branches.
101      '''
102      return self.Create(file_system,
103                         SingleFile(lambda _, data:
104                             json_parse.Parse(ToUnicode(data))),
105                         CompiledFileSystem,
106                         category='json')
107
108    @memoize
109    def ForAPISchema(self, file_system):
110      '''Creates a CompiledFileSystem for parsing raw JSON or IDL API schema
111      data and formatting it so that it can be used by other classes, such
112      as Model and APISchemaGraph.
113      '''
114      return self.Create(file_system,
115                         SingleFile(Unicode(schema_util.ProcessSchema)),
116                         CompiledFileSystem,
117                         category='api-schema')
118
119    @memoize
120    def ForTemplates(self, file_system):
121      '''Creates a CompiledFileSystem for parsing templates.
122      '''
123      return self.Create(
124          file_system,
125          SingleFile(lambda path, text: Handlebar(ToUnicode(text), name=path)),
126          CompiledFileSystem)
127
128    @memoize
129    def ForUnicode(self, file_system):
130      '''Creates a CompiledFileSystem for Unicode text processing.
131      '''
132      return self.Create(
133        file_system,
134        SingleFile(lambda _, text: ToUnicode(text)),
135        CompiledFileSystem,
136        category='text')
137
138  def __init__(self,
139               file_system,
140               compilation_function,
141               file_object_store,
142               list_object_store):
143    self._file_system = file_system
144    self._compilation_function = compilation_function
145    self._file_object_store = file_object_store
146    self._list_object_store = list_object_store
147
148  def _RecursiveList(self, path):
149    '''Returns a Future containing the recursive directory listing of |path| as
150    a flat list of paths.
151    '''
152    def split_dirs_from_files(paths):
153      '''Returns a tuple (dirs, files) where |dirs| contains the directory
154      names in |paths| and |files| contains the files.
155      '''
156      result = [], []
157      for path in paths:
158        result[0 if path.endswith('/') else 1].append(path)
159      return result
160
161    def add_prefix(prefix, paths):
162      return [prefix + path for path in paths]
163
164    # Read in the initial list of files. Do this eagerly (i.e. not part of the
165    # asynchronous Future contract) because there's a greater chance to
166    # parallelise fetching with the second layer (can fetch multiple paths).
167    try:
168      first_layer_dirs, first_layer_files = split_dirs_from_files(
169          self._file_system.ReadSingle(path).Get())
170    except FileNotFoundError:
171      return Future(exc_info=sys.exc_info())
172
173    if not first_layer_dirs:
174      return Future(value=first_layer_files)
175
176    second_layer_listing = self._file_system.Read(
177        add_prefix(path, first_layer_dirs))
178
179    def resolve():
180      def get_from_future_listing(futures):
181        '''Recursively lists files from directory listing |futures|.
182        '''
183        dirs, files = [], []
184        for dir_name, listing in futures.Get().iteritems():
185          new_dirs, new_files = split_dirs_from_files(listing)
186          # |dirs| are paths for reading. Add the full prefix relative to
187          # |path| so that |file_system| can find the files.
188          dirs += add_prefix(dir_name, new_dirs)
189          # |files| are not for reading, they are for returning to the caller.
190          # This entire function set (i.e. GetFromFileListing) is defined to
191          # not include the fetched-path in the result, however, |dir_name|
192          # will be prefixed with |path|. Strip it.
193          assert dir_name.startswith(path)
194          files += add_prefix(dir_name[len(path):], new_files)
195        if dirs:
196          files += get_from_future_listing(self._file_system.Read(dirs))
197        return files
198
199      return first_layer_files + get_from_future_listing(second_layer_listing)
200
201    return Future(callback=resolve)
202
203  def GetFromFile(self, path):
204    '''Calls |compilation_function| on the contents of the file at |path|.  If
205    |binary| is True then the file will be read as binary - but this will only
206    apply for the first time the file is fetched; if already cached, |binary|
207    will be ignored.
208    '''
209    AssertIsFile(path)
210
211    try:
212      version = self._file_system.Stat(path).version
213    except FileNotFoundError:
214      return Future(exc_info=sys.exc_info())
215
216    cache_entry = self._file_object_store.Get(path).Get()
217    if (cache_entry is not None) and (version == cache_entry.version):
218      return Future(value=cache_entry._cache_data)
219
220    future_files = self._file_system.ReadSingle(path)
221    def resolve():
222      cache_data = self._compilation_function(path, future_files.Get())
223      self._file_object_store.Set(path, _CacheEntry(cache_data, version))
224      return cache_data
225    return Future(callback=resolve)
226
227  def GetFromFileListing(self, path):
228    '''Calls |compilation_function| on the listing of the files at |path|.
229    Assumes that the path given is to a directory.
230    '''
231    AssertIsDirectory(path)
232
233    try:
234      version = self._file_system.Stat(path).version
235    except FileNotFoundError:
236      return Future(exc_info=sys.exc_info())
237
238    cache_entry = self._list_object_store.Get(path).Get()
239    if (cache_entry is not None) and (version == cache_entry.version):
240      return Future(value=cache_entry._cache_data)
241
242    recursive_list_future = self._RecursiveList(path)
243    def resolve():
244      cache_data = self._compilation_function(path, recursive_list_future.Get())
245      self._list_object_store.Set(path, _CacheEntry(cache_data, version))
246      return cache_data
247    return Future(callback=resolve)
248
249  def GetFileVersion(self, path):
250    cache_entry = self._file_object_store.Get(path).Get()
251    if cache_entry is not None:
252      return cache_entry.version
253    return self._file_system.Stat(path).version
254
255  def GetFileListingVersion(self, path):
256    if not path.endswith('/'):
257      path += '/'
258    cache_entry = self._list_object_store.Get(path).Get()
259    if cache_entry is not None:
260      return cache_entry.version
261    return self._file_system.Stat(path).version
262
263  def FileExists(self, path):
264    return self._file_system.Exists(path)
265
266  def GetIdentity(self):
267    return self._file_system.GetIdentity()
268