• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import json
6import logging
7from StringIO import StringIO
8import posixpath
9
10from appengine_blobstore import AppEngineBlobstore, BLOBSTORE_GITHUB
11from appengine_url_fetcher import AppEngineUrlFetcher
12from appengine_wrappers import urlfetch, blobstore
13from docs_server_utils import StringIdentity
14from file_system import FileSystem, StatInfo
15from future import Future
16from path_util import IsDirectory
17import url_constants
18from zipfile import ZipFile, BadZipfile
19
20ZIP_KEY = 'zipball'
21USERNAME = None
22PASSWORD = None
23
24
25def _MakeBlobstoreKey(version):
26  return ZIP_KEY + '.' + str(version)
27
28
29def _GetAsyncFetchCallback(fetcher,
30                           username,
31                           password,
32                           blobstore,
33                           key_to_set,
34                           key_to_delete=None):
35  fetch = fetcher.FetchAsync(ZIP_KEY, username=username, password=password)
36
37  def resolve():
38    try:
39      result = fetch.Get()
40      # Check if Github authentication failed.
41      if result.status_code == 401:
42        logging.error('Github authentication failed for %s, falling back to '
43                      'unauthenticated.' % USERNAME)
44        blob = fetcher.Fetch(ZIP_KEY).content
45      else:
46        blob = result.content
47    except urlfetch.DownloadError as e:
48      logging.error('Bad github zip file: %s' % e)
49      return None
50    if key_to_delete is not None:
51      blobstore.Delete(_MakeBlobstoreKey(key_to_delete, BLOBSTORE_GITHUB))
52    try:
53      return_zip = ZipFile(StringIO(blob))
54    except BadZipfile as e:
55      logging.error('Bad github zip file: %s' % e)
56      return None
57
58    blobstore.Set(_MakeBlobstoreKey(key_to_set), blob, BLOBSTORE_GITHUB)
59    return return_zip
60
61  return resolve
62
63
64class GithubFileSystem(FileSystem):
65  @staticmethod
66  def CreateChromeAppsSamples(object_store_creator):
67    return GithubFileSystem(
68        '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS,
69        AppEngineBlobstore(),
70        object_store_creator)
71
72  def __init__(self, url, blobstore, object_store_creator):
73    # If we key the password store on the app version then the whole advantage
74    # of having it in the first place is greatly lessened (likewise it should
75    # always start populated).
76    password_store = object_store_creator.Create(
77        GithubFileSystem,
78        app_version=None,
79        category='password',
80        start_empty=False)
81    if USERNAME is None:
82      password_data = password_store.GetMulti(('username', 'password')).Get()
83      self._username, self._password = (password_data.get('username'),
84                                        password_data.get('password'))
85    else:
86      password_store.SetMulti({'username': USERNAME, 'password': PASSWORD})
87      self._username, self._password = (USERNAME, PASSWORD)
88
89    self._url = url
90    self._fetcher = AppEngineUrlFetcher(url)
91    self._blobstore = blobstore
92    self._stat_object_store = object_store_creator.Create(GithubFileSystem)
93    self._version = None
94    self._GetZip(self.Stat(ZIP_KEY).version)
95
96  def _GetZip(self, version):
97    try:
98      blob = self._blobstore.Get(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
99    except blobstore.BlobNotFoundError:
100      self._zip_file = Future(value=None)
101      return
102    if blob is not None:
103      try:
104        self._zip_file = Future(value=ZipFile(StringIO(blob)))
105      except BadZipfile as e:
106        self._blobstore.Delete(_MakeBlobstoreKey(version), BLOBSTORE_GITHUB)
107        logging.error('Bad github zip file: %s' % e)
108        self._zip_file = Future(value=None)
109    else:
110      self._zip_file = Future(
111          callback=_GetAsyncFetchCallback(self._fetcher,
112                                          self._username,
113                                          self._password,
114                                          self._blobstore,
115                                          version,
116                                          key_to_delete=self._version))
117    self._version = version
118
119  def _ReadFile(self, path):
120    try:
121      zip_file = self._zip_file.Get()
122    except Exception as e:
123      logging.error('Github ReadFile error: %s' % e)
124      return ''
125    if zip_file is None:
126      logging.error('Bad github zip file.')
127      return ''
128    prefix = zip_file.namelist()[0]
129    return zip_file.read(prefix + path)
130
131  def _ListDir(self, path):
132    try:
133      zip_file = self._zip_file.Get()
134    except Exception as e:
135      logging.error('Github ListDir error: %s' % e)
136      return []
137    if zip_file is None:
138      logging.error('Bad github zip file.')
139      return []
140    filenames = zip_file.namelist()
141    # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
142    filenames = [f[len(filenames[0]):] for f in filenames]
143    # Remove the path of the directory we're listing from the filenames.
144    filenames = [f[len(path):] for f in filenames
145                 if f != path and f.startswith(path)]
146    # Remove all files not directly in this directory.
147    return [f for f in filenames if f[:-1].count('/') == 0]
148
149  def Read(self, paths, skip_not_found=False):
150    version = self.Stat(ZIP_KEY).version
151    if version != self._version:
152      self._GetZip(version)
153    result = {}
154    for path in paths:
155      if IsDirectory(path):
156        result[path] = self._ListDir(path)
157      else:
158        result[path] = self._ReadFile(path)
159    return Future(value=result)
160
161  def _DefaultStat(self, path):
162    version = 0
163    # TODO(kalman): we should replace all of this by wrapping the
164    # GithubFileSystem in a CachingFileSystem. A lot of work has been put into
165    # CFS to be robust, and GFS is missing out.
166    # For example: the following line is wrong, but it could be moot.
167    self._stat_object_store.Set(path, version)
168    return StatInfo(version)
169
170  def Stat(self, path):
171    version = self._stat_object_store.Get(path).Get()
172    if version is not None:
173      return StatInfo(version)
174    try:
175      result = self._fetcher.Fetch('commits/HEAD',
176                                   username=USERNAME,
177                                   password=PASSWORD)
178    except urlfetch.DownloadError as e:
179      logging.warning('GithubFileSystem Stat: %s' % e)
180      return self._DefaultStat(path)
181
182    # Check if Github authentication failed.
183    if result.status_code == 401:
184      logging.warning('Github authentication failed for %s, falling back to '
185                      'unauthenticated.' % USERNAME)
186      try:
187        result = self._fetcher.Fetch('commits/HEAD')
188      except urlfetch.DownloadError as e:
189        logging.warning('GithubFileSystem Stat: %s' % e)
190        return self._DefaultStat(path)
191
192    # Parse response JSON - but sometimes github gives us invalid JSON.
193    try:
194      version = json.loads(result.content)['sha']
195      self._stat_object_store.Set(path, version)
196      return StatInfo(version)
197    except StandardError as e:
198      logging.warning(
199          ('%s: got invalid or unexpected JSON from github. Response status ' +
200           'was %s, content %s') % (e, result.status_code, result.content))
201      return self._DefaultStat(path)
202
203  def GetIdentity(self):
204    return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url))
205