1# Copyright 2018 Google Inc. All rights reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import datetime 16import inspect 17import logging 18import json 19 20import endpoints 21from protorpc import messages 22from protorpc import remote 23from google.appengine.ext import ndb 24 25from webapp.src import vtslab_status as Status 26from webapp.src.proto import model 27 28MAX_QUERY_SIZE = 1000 29 30COUNT_REQUEST_RESOURCE = endpoints.ResourceContainer(model.CountRequestMessage) 31GET_REQUEST_RESOURCE = endpoints.ResourceContainer(model.GetRequestMessage) 32 33 34class EndpointBase(remote.Service): 35 """A base class for endpoint implementation.""" 36 37 def GetCommonAttributes(self, resource, reference): 38 """Gets a list of common attribute names. 39 40 This method finds the attributes assigned in 'resource' instance, and 41 filters out if the attributes are not a member of 'reference' class. 42 43 Args: 44 resource: either a protorpc.messages.Message instance, 45 or a ndb.Model instance. 46 reference: either a protorpc.messages.Message class, 47 or a ndb.Model class. 48 49 Returns: 50 a list of string, attribute names exist on resource and reference. 51 52 Raises: 53 ValueError if resource or reference is not supported class. 54 """ 55 # check resource type and absorb list of assigned attributes. 56 resource_attrs = self.GetAttributes(resource, assigned_only=True) 57 reference_attrs = self.GetAttributes(reference) 58 return [x for x in resource_attrs if x in reference_attrs] 59 60 def GetAttributes(self, value, assigned_only=False): 61 """Gets a list of attributes. 62 63 Args: 64 value: a class instance or a class itself. 65 assigned_only: True to get only assigned attributes when value is 66 an instance, False to get all attributes. 67 68 Raises: 69 ValueError if value is not supported class. 70 """ 71 attrs = [] 72 if inspect.isclass(value): 73 if assigned_only: 74 logging.warning( 75 "Please use a class instance for 'resource' argument.") 76 77 if (issubclass(value, messages.Message) 78 or issubclass(value, ndb.Model)): 79 attrs = [ 80 x[0] for x in value.__dict__.items() 81 if not x[0].startswith("_") 82 ] 83 else: 84 raise ValueError("Only protorpc.messages.Message or ndb.Model " 85 "class are supported.") 86 else: 87 if isinstance(value, messages.Message): 88 attrs = [ 89 x.name for x in value.all_fields() 90 if not assigned_only or ( 91 value.get_assigned_value(x.name) not in [None, []]) 92 ] 93 elif isinstance(value, ndb.Model): 94 attrs = [ 95 x for x in list(value.to_dict()) 96 if not assigned_only or ( 97 getattr(value, x, None) not in [None, []]) 98 ] 99 else: 100 raise ValueError("Only protorpc.messages.Message or ndb.Model " 101 "class are supported.") 102 103 return attrs 104 105 def Count(self, metaclass, filters=None): 106 """Counts entities from datastore with options. 107 108 Args: 109 metaclass: a metaclass for ndb model. 110 filters: a list of tuples. Each tuple consists of three values: 111 key, method, and value. 112 113 Returns: 114 a number of entities. 115 """ 116 query, _ = self.CreateQueryFilter(metaclass=metaclass, filters=filters) 117 return query.count() 118 119 def Fetch(self, 120 metaclass, 121 size, 122 offset=0, 123 filters=None, 124 sort_key="", 125 direction="asc"): 126 """Fetches entities from datastore with options. 127 128 Args: 129 metaclass: a metaclass for ndb model. 130 size: an integer, max number of entities to fetch at once. 131 offset: an integer, number of query results to skip. 132 filters: a list of filter tuple, a form of (key: string, 133 method: integer, value: string). 134 sort_key: a string, key name to sort by. 135 direction: a string, "asc" for ascending order and "desc" for 136 descending order. 137 138 Returns: 139 a list of fetched entities. 140 a boolean, True if there is next page or False if not. 141 """ 142 query, empty_repeated_field = self.CreateQueryFilter( 143 metaclass=metaclass, filters=filters) 144 sorted_query = self.SortQuery( 145 query=query, 146 metaclass=metaclass, 147 sort_key=sort_key, 148 direction=direction) 149 150 if size: 151 entities, _, more = sorted_query.fetch_page( 152 page_size=size, offset=offset) 153 else: 154 entities = sorted_query.fetch() 155 more = False 156 157 if empty_repeated_field: 158 entities = [ 159 x for x in entities 160 if all([not getattr(x, attr) for attr in empty_repeated_field]) 161 ] 162 163 return entities, more 164 165 def CreateQueryFilter(self, metaclass, filters): 166 """Creates a query with the given filters. 167 168 Args: 169 metaclass: a metaclass for ndb model. 170 filters: a list of tuples. Each tuple consists of three values: 171 key, method, and value. 172 173 Returns: 174 a filtered query for the given metaclass. 175 a list of strings that failed to create the query due to its empty 176 value for the repeated property. 177 """ 178 empty_repeated_field = [] 179 query = metaclass.query() 180 if not filters: 181 return query, empty_repeated_field 182 183 for _filter in filters: 184 property_key = _filter["key"] 185 method = _filter["method"] 186 value = _filter["value"] 187 if type(value) is str or type(value) is unicode: 188 if isinstance(metaclass._properties[property_key], 189 ndb.BooleanProperty): 190 value = value.lower() in ("yes", "true", "1") 191 elif isinstance(metaclass._properties[property_key], 192 ndb.IntegerProperty): 193 value = int(value) 194 if metaclass._properties[property_key]._repeated: 195 if value: 196 value = [value] 197 if method == Status.FILTER_METHOD[Status.FILTER_Has]: 198 query = query.filter( 199 getattr(metaclass, property_key).IN(value)) 200 else: 201 logging.warning( 202 "You cannot compare repeated " 203 "properties except 'IN(has)' operation.") 204 else: 205 logging.debug("Empty repeated list cannot be queried.") 206 empty_repeated_field.append(value) 207 elif isinstance(metaclass._properties[property_key], 208 ndb.DateTimeProperty): 209 if method == Status.FILTER_METHOD[Status.FILTER_LessThan]: 210 query = query.filter( 211 getattr(metaclass, property_key) < datetime.datetime. 212 now() - datetime.timedelta(hours=int(value))) 213 elif method == Status.FILTER_METHOD[Status.FILTER_GreaterThan]: 214 query = query.filter( 215 getattr(metaclass, property_key) > datetime.datetime. 216 now() - datetime.timedelta(hours=int(value))) 217 else: 218 logging.debug("DateTimeProperty only allows <=(less than) " 219 "and >=(greater than) operation.") 220 else: 221 if method == Status.FILTER_METHOD[Status.FILTER_EqualTo]: 222 query = query.filter( 223 getattr(metaclass, property_key) == value) 224 elif method == Status.FILTER_METHOD[Status.FILTER_LessThan]: 225 query = query.filter( 226 getattr(metaclass, property_key) < value) 227 elif method == Status.FILTER_METHOD[Status.FILTER_GreaterThan]: 228 query = query.filter( 229 getattr(metaclass, property_key) > value) 230 elif method == Status.FILTER_METHOD[ 231 Status.FILTER_LessThanOrEqualTo]: 232 query = query.filter( 233 getattr(metaclass, property_key) <= value) 234 elif method == Status.FILTER_METHOD[ 235 Status.FILTER_GreaterThanOrEqualTo]: 236 query = query.filter( 237 getattr(metaclass, property_key) >= value) 238 elif method == Status.FILTER_METHOD[Status.FILTER_NotEqualTo]: 239 query = query.filter( 240 getattr(metaclass, property_key) != value).order( 241 getattr(metaclass, property_key), metaclass.key) 242 elif method == Status.FILTER_METHOD[Status.FILTER_Has]: 243 query = query.filter( 244 getattr(metaclass, property_key).IN(value)).order( 245 getattr(metaclass, property_key), metaclass.key) 246 else: 247 logging.warning( 248 "{} is not supported filter method.".format(method)) 249 return query, empty_repeated_field 250 251 def SortQuery(self, query, metaclass, sort_key, direction): 252 """Sorts the given query with sort_key and direction. 253 254 Args: 255 query: a ndb query to sort. 256 metaclass: a metaclass for ndb model. 257 sort_key: a string, key name to sort by. 258 direction: a string, "asc" for ascending order and "desc" for 259 descending order. 260 """ 261 if sort_key: 262 if direction == "desc": 263 query = query.order(-getattr(metaclass, sort_key)) 264 else: 265 query = query.order(getattr(metaclass, sort_key)) 266 267 return query 268 269 def CreateFilterList(self, filter_string, metaclass): 270 """Creates a list of filters. 271 272 Args: 273 filter_string: a string, stringified JSON which contains 'key', 274 'method', 'value' to build filter information. 275 metaclass: a metaclass for ndb model. 276 277 Returns: 278 a list of tuples where each tuple consists of three values: 279 key, method, and value. 280 """ 281 model_properties = self.GetAttributes(metaclass) 282 filters = [] 283 if filter_string: 284 filters = json.loads(filter_string) 285 for _filter in filters: 286 if _filter["key"] not in model_properties: 287 filters.remove(_filter) 288 return filters 289 290 def Get(self, request, metaclass, message): 291 """Handles a request through /get endpoints API to retrieves entities. 292 293 Args: 294 request: a request body message received through /get API. 295 metaclass: a metaclass for ndb model. This method will fetch the 296 'metaclass' type of model from datastore. 297 message: a Protocol RPC message class. Fetched entities will be 298 converted to this message class instances. 299 300 Returns: 301 a list of fetched entities. 302 a boolean, True if there is next page or False if not. 303 """ 304 size = request.size if request.size else MAX_QUERY_SIZE 305 offset = request.offset if request.offset else 0 306 307 filters = self.CreateFilterList( 308 filter_string=request.filter, metaclass=metaclass) 309 310 entities, more = self.Fetch( 311 metaclass=metaclass, 312 size=size, 313 filters=filters, 314 offset=offset, 315 sort_key=request.sort, 316 direction=request.direction, 317 ) 318 319 return_list = [] 320 for entity in entities: 321 entity_dict = {} 322 assigned_attributes = self.GetCommonAttributes( 323 resource=entity, reference=message) 324 for attr in assigned_attributes: 325 entity_dict[attr] = getattr(entity, attr, None) 326 if hasattr(message, "urlsafe_key"): 327 entity_dict["urlsafe_key"] = entity.key.urlsafe() 328 return_list.append(entity_dict) 329 330 return return_list, more 331