1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import json 6import logging 7from StringIO import StringIO 8 9import appengine_blobstore as blobstore 10from appengine_url_fetcher import AppEngineUrlFetcher 11from appengine_wrappers import urlfetch 12from docs_server_utils import StringIdentity 13from file_system import FileSystem, StatInfo 14from future import Future 15import url_constants 16from zipfile import ZipFile, BadZipfile 17 18ZIP_KEY = 'zipball' 19USERNAME = None 20PASSWORD = None 21 22def _MakeBlobstoreKey(version): 23 return ZIP_KEY + '.' + str(version) 24 25class _AsyncFetchFutureZip(object): 26 def __init__(self, 27 fetcher, 28 username, 29 password, 30 blobstore, 31 key_to_set, 32 key_to_delete=None): 33 self._fetcher = fetcher 34 self._fetch = fetcher.FetchAsync(ZIP_KEY, 35 username=username, 36 password=password) 37 self._blobstore = blobstore 38 self._key_to_set = key_to_set 39 self._key_to_delete = key_to_delete 40 41 def Get(self): 42 try: 43 result = self._fetch.Get() 44 # Check if Github authentication failed. 45 if result.status_code == 401: 46 logging.error('Github authentication failed for %s, falling back to ' 47 'unauthenticated.' % USERNAME) 48 blob = self._fetcher.Fetch(ZIP_KEY).content 49 else: 50 blob = result.content 51 except urlfetch.DownloadError as e: 52 logging.error('Bad github zip file: %s' % e) 53 return None 54 if self._key_to_delete is not None: 55 self._blobstore.Delete(_MakeBlobstoreKey(self._key_to_delete), 56 blobstore.BLOBSTORE_GITHUB) 57 try: 58 return_zip = ZipFile(StringIO(blob)) 59 except BadZipfile as e: 60 logging.error('Bad github zip file: %s' % e) 61 return None 62 63 self._blobstore.Set(_MakeBlobstoreKey(self._key_to_set), 64 blob, 65 blobstore.BLOBSTORE_GITHUB) 66 return return_zip 67 68class GithubFileSystem(FileSystem): 69 @staticmethod 70 def CreateChromeAppsSamples(object_store_creator): 71 return GithubFileSystem( 72 '%s/GoogleChrome/chrome-app-samples' % url_constants.GITHUB_REPOS, 73 blobstore.AppEngineBlobstore(), 74 object_store_creator) 75 76 def __init__(self, url, blobstore, object_store_creator): 77 # If we key the password store on the app version then the whole advantage 78 # of having it in the first place is greatly lessened (likewise it should 79 # always start populated). 80 password_store = object_store_creator.Create( 81 GithubFileSystem, 82 app_version=None, 83 category='password', 84 start_empty=False) 85 if USERNAME is None: 86 password_data = password_store.GetMulti(('username', 'password')).Get() 87 self._username, self._password = (password_data.get('username'), 88 password_data.get('password')) 89 else: 90 password_store.SetMulti({'username': USERNAME, 'password': PASSWORD}) 91 self._username, self._password = (USERNAME, PASSWORD) 92 93 self._url = url 94 self._fetcher = AppEngineUrlFetcher(url) 95 self._blobstore = blobstore 96 self._stat_object_store = object_store_creator.Create(GithubFileSystem) 97 self._version = None 98 self._GetZip(self.Stat(ZIP_KEY).version) 99 100 def _GetZip(self, version): 101 try: 102 blob = self._blobstore.Get(_MakeBlobstoreKey(version), 103 blobstore.BLOBSTORE_GITHUB) 104 except: 105 self._zip_file = Future(value=None) 106 return 107 if blob is not None: 108 try: 109 self._zip_file = Future(value=ZipFile(StringIO(blob))) 110 except BadZipfile as e: 111 self._blobstore.Delete(_MakeBlobstoreKey(version), 112 blobstore.BLOBSTORE_GITHUB) 113 logging.error('Bad github zip file: %s' % e) 114 self._zip_file = Future(value=None) 115 else: 116 self._zip_file = Future( 117 delegate=_AsyncFetchFutureZip(self._fetcher, 118 self._username, 119 self._password, 120 self._blobstore, 121 version, 122 key_to_delete=self._version)) 123 self._version = version 124 125 def _ReadFile(self, path): 126 try: 127 zip_file = self._zip_file.Get() 128 except Exception as e: 129 logging.error('Github ReadFile error: %s' % e) 130 return '' 131 if zip_file is None: 132 logging.error('Bad github zip file.') 133 return '' 134 prefix = zip_file.namelist()[0][:-1] 135 return zip_file.read(prefix + path) 136 137 def _ListDir(self, path): 138 try: 139 zip_file = self._zip_file.Get() 140 except Exception as e: 141 logging.error('Github ListDir error: %s' % e) 142 return [] 143 if zip_file is None: 144 logging.error('Bad github zip file.') 145 return [] 146 filenames = zip_file.namelist() 147 # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f) 148 filenames = [f[len(filenames[0]) - 1:] for f in filenames] 149 # Remove the path of the directory we're listing from the filenames. 150 filenames = [f[len(path):] for f in filenames 151 if f != path and f.startswith(path)] 152 # Remove all files not directly in this directory. 153 return [f for f in filenames if f[:-1].count('/') == 0] 154 155 def Read(self, paths): 156 version = self.Stat(ZIP_KEY).version 157 if version != self._version: 158 self._GetZip(version) 159 result = {} 160 for path in paths: 161 if path.endswith('/'): 162 result[path] = self._ListDir(path) 163 else: 164 result[path] = self._ReadFile(path) 165 return Future(value=result) 166 167 def _DefaultStat(self, path): 168 version = 0 169 # TODO(kalman): we should replace all of this by wrapping the 170 # GithubFileSystem in a CachingFileSystem. A lot of work has been put into 171 # CFS to be robust, and GFS is missing out. 172 # For example: the following line is wrong, but it could be moot. 173 self._stat_object_store.Set(path, version) 174 return StatInfo(version) 175 176 def Stat(self, path): 177 version = self._stat_object_store.Get(path).Get() 178 if version is not None: 179 return StatInfo(version) 180 try: 181 result = self._fetcher.Fetch('commits/HEAD', 182 username=USERNAME, 183 password=PASSWORD) 184 except urlfetch.DownloadError as e: 185 logging.warning('GithubFileSystem Stat: %s' % e) 186 return self._DefaultStat(path) 187 188 # Check if Github authentication failed. 189 if result.status_code == 401: 190 logging.warning('Github authentication failed for %s, falling back to ' 191 'unauthenticated.' % USERNAME) 192 try: 193 result = self._fetcher.Fetch('commits/HEAD') 194 except urlfetch.DownloadError as e: 195 logging.warning('GithubFileSystem Stat: %s' % e) 196 return self._DefaultStat(path) 197 198 # Parse response JSON - but sometimes github gives us invalid JSON. 199 try: 200 version = json.loads(result.content)['sha'] 201 self._stat_object_store.Set(path, version) 202 return StatInfo(version) 203 except StandardError as e: 204 logging.warning( 205 ('%s: got invalid or unexpected JSON from github. Response status ' + 206 'was %s, content %s') % (e, result.status_code, result.content)) 207 return self._DefaultStat(path) 208 209 def GetIdentity(self): 210 return '%s@%s' % (self.__class__.__name__, StringIdentity(self._url)) 211