1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Base Cloud API Client. 17 18BasicCloudApiCliend does basic setup for a cloud API. 19""" 20import httplib 21import logging 22import socket 23import ssl 24 25# pylint: disable=import-error 26from apiclient import errors as gerrors 27from apiclient.discovery import build 28import apiclient.http 29import httplib2 30from oauth2client import client 31 32from acloud import errors 33from acloud.internal.lib import utils 34 35logger = logging.getLogger(__name__) 36 37 38class BaseCloudApiClient(object): 39 """A class that does basic setup for a cloud API.""" 40 41 # To be overriden by subclasses. 42 API_NAME = "" 43 API_VERSION = "v1" 44 SCOPE = "" 45 46 # Defaults for retry. 47 RETRY_COUNT = 5 48 RETRY_BACKOFF_FACTOR = 1.5 49 RETRY_SLEEP_MULTIPLIER = 2 50 RETRY_HTTP_CODES = [ 51 # 403 is to retry the "Rate Limit Exceeded" error. 52 # We could retry on a finer-grained error message later if necessary. 53 403, 54 500, # Internal Server Error 55 502, # Bad Gateway 56 503, # Service Unavailable 57 ] 58 RETRIABLE_ERRORS = (httplib.HTTPException, httplib2.HttpLib2Error, 59 socket.error, ssl.SSLError) 60 RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) 61 62 def __init__(self, oauth2_credentials): 63 """Initialize. 64 65 Args: 66 oauth2_credentials: An oauth2client.OAuth2Credentials instance. 67 """ 68 self._service = self.InitResourceHandle(oauth2_credentials) 69 70 @classmethod 71 def InitResourceHandle(cls, oauth2_credentials): 72 """Authenticate and initialize a Resource object. 73 74 Authenticate http and create a Resource object with methods 75 for interacting with the service. 76 77 Args: 78 oauth2_credentials: An oauth2client.OAuth2Credentials instance. 79 80 Returns: 81 An apiclient.discovery.Resource object 82 """ 83 http_auth = oauth2_credentials.authorize(httplib2.Http()) 84 return utils.RetryExceptionType( 85 exception_types=cls.RETRIABLE_AUTH_ERRORS, 86 max_retries=cls.RETRY_COUNT, 87 functor=build, 88 sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER, 89 retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR, 90 serviceName=cls.API_NAME, 91 version=cls.API_VERSION, 92 # This is workaround for a known issue of some veriosn 93 # of api client. 94 # https://github.com/google/google-api-python-client/issues/435 95 cache_discovery=False, 96 http=http_auth) 97 98 @staticmethod 99 def _ShouldRetry(exception, retry_http_codes, 100 other_retriable_errors): 101 """Check if exception is retriable. 102 103 Args: 104 exception: An instance of Exception. 105 retry_http_codes: a list of integers, retriable HTTP codes of 106 HttpError 107 other_retriable_errors: a tuple of error types to retry other than 108 HttpError. 109 110 Returns: 111 Boolean, True if retriable, False otherwise. 112 """ 113 if isinstance(exception, other_retriable_errors): 114 return True 115 116 if isinstance(exception, errors.HttpError): 117 if exception.code in retry_http_codes: 118 return True 119 else: 120 logger.debug("_ShouldRetry: Exception code %s not in %s: %s", 121 exception.code, retry_http_codes, str(exception)) 122 123 logger.debug("_ShouldRetry: Exception %s is not one of %s: %s", 124 type(exception), 125 list(other_retriable_errors) + [errors.HttpError], 126 str(exception)) 127 return False 128 129 @staticmethod 130 def _TranslateError(exception): 131 """Translate the exception to a desired type. 132 133 Args: 134 exception: An instance of Exception. 135 136 Returns: 137 gerrors.HttpError will be translated to errors.HttpError. 138 If the error code is errors.HTTP_NOT_FOUND_CODE, it will 139 be translated to errors.ResourceNotFoundError. 140 Unrecognized error type will not be translated and will 141 be returned as is. 142 """ 143 if isinstance(exception, gerrors.HttpError): 144 exception = errors.HttpError.CreateFromHttpError(exception) 145 if exception.code == errors.HTTP_NOT_FOUND_CODE: 146 exception = errors.ResourceNotFoundError( 147 exception.code, str(exception)) 148 return exception 149 150 def ExecuteOnce(self, api): 151 """Execute an api and parse the errors. 152 153 Args: 154 api: An apiclient.http.HttpRequest, representing the api to execute. 155 156 Returns: 157 Execution result of the api. 158 159 Raises: 160 errors.ResourceNotFoundError: For 404 error. 161 errors.HttpError: For other types of http error. 162 """ 163 try: 164 return api.execute() 165 except gerrors.HttpError as e: 166 raise self._TranslateError(e) 167 168 def Execute(self, 169 api, 170 retry_http_codes=None, 171 max_retry=None, 172 sleep=None, 173 backoff_factor=None, 174 other_retriable_errors=None): 175 """Execute an api with retry. 176 177 Call ExecuteOnce and retry on http error with given codes. 178 179 Args: 180 api: An apiclient.http.HttpRequest, representing the api to execute: 181 retry_http_codes: A list of http codes to retry. 182 max_retry: See utils.Retry. 183 sleep: See utils.Retry. 184 backoff_factor: See utils.Retry. 185 other_retriable_errors: A tuple of error types that should be retried 186 other than errors.HttpError. 187 188 Returns: 189 Execution result of the api. 190 191 Raises: 192 See ExecuteOnce. 193 """ 194 retry_http_codes = (self.RETRY_HTTP_CODES 195 if retry_http_codes is None else retry_http_codes) 196 max_retry = (self.RETRY_COUNT if max_retry is None else max_retry) 197 sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep) 198 backoff_factor = (self.RETRY_BACKOFF_FACTOR 199 if backoff_factor is None else backoff_factor) 200 other_retriable_errors = (self.RETRIABLE_ERRORS 201 if other_retriable_errors is None else 202 other_retriable_errors) 203 204 def _Handler(exc): 205 """Check if |exc| is a retriable exception. 206 207 Args: 208 exc: An exception. 209 210 Returns: 211 True if exc is an errors.HttpError and code exists in |retry_http_codes| 212 False otherwise. 213 """ 214 if self._ShouldRetry(exc, retry_http_codes, 215 other_retriable_errors): 216 logger.debug("Will retry error: %s", str(exc)) 217 return True 218 return False 219 220 return utils.Retry( 221 _Handler, 222 max_retries=max_retry, 223 functor=self.ExecuteOnce, 224 sleep_multiplier=sleep, 225 retry_backoff_factor=backoff_factor, 226 api=api) 227 228 def BatchExecuteOnce(self, requests): 229 """Execute requests in a batch. 230 231 Args: 232 requests: A dictionary where key is request id and value 233 is an http request. 234 235 Returns: 236 results, a dictionary in the following format 237 {request_id: (response, exception)} 238 request_ids are those from requests; response 239 is the http response for the request or None on error; 240 exception is an instance of DriverError or None if no error. 241 """ 242 results = {} 243 244 def _CallBack(request_id, response, exception): 245 results[request_id] = (response, self._TranslateError(exception)) 246 247 batch = apiclient.http.BatchHttpRequest() 248 for request_id, request in requests.iteritems(): 249 batch.add( 250 request=request, callback=_CallBack, request_id=request_id) 251 batch.execute() 252 return results 253 254 def BatchExecute(self, 255 requests, 256 retry_http_codes=None, 257 max_retry=None, 258 sleep=None, 259 backoff_factor=None, 260 other_retriable_errors=None): 261 """Batch execute multiple requests with retry. 262 263 Call BatchExecuteOnce and retry on http error with given codes. 264 265 Args: 266 requests: A dictionary where key is request id picked by caller, 267 and value is a apiclient.http.HttpRequest. 268 retry_http_codes: A list of http codes to retry. 269 max_retry: See utils.Retry. 270 sleep: See utils.Retry. 271 backoff_factor: See utils.Retry. 272 other_retriable_errors: A tuple of error types that should be retried 273 other than errors.HttpError. 274 275 Returns: 276 results, a dictionary in the following format 277 {request_id: (response, exception)} 278 request_ids are those from requests; response 279 is the http response for the request or None on error; 280 exception is an instance of DriverError or None if no error. 281 """ 282 executor = utils.BatchHttpRequestExecutor( 283 self.BatchExecuteOnce, 284 requests=requests, 285 retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES, 286 max_retry=max_retry or self.RETRY_COUNT, 287 sleep=sleep or self.RETRY_SLEEP_MULTIPLIER, 288 backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR, 289 other_retriable_errors=other_retriable_errors 290 or self.RETRIABLE_ERRORS) 291 executor.Execute() 292 return executor.GetResults() 293 294 def ListWithMultiPages(self, api_resource, *args, **kwargs): 295 """Call an api that list a type of resource. 296 297 Multiple google services support listing a type of 298 resource (e.g list gce instances, list storage objects). 299 The querying pattern is similar -- 300 Step 1: execute the api and get a response object like, 301 { 302 "items": [..list of resource..], 303 # The continuation token that can be used 304 # to get the next page. 305 "nextPageToken": "A String", 306 } 307 Step 2: execute the api again with the nextPageToken to 308 retrieve more pages and get a response object. 309 310 Step 3: Repeat Step 2 until no more page. 311 312 This method encapsulates the generic logic of 313 calling such listing api. 314 315 Args: 316 api_resource: An apiclient.discovery.Resource object 317 used to create an http request for the listing api. 318 *args: Arguments used to create the http request. 319 **kwargs: Keyword based arguments to create the http 320 request. 321 322 Returns: 323 A list of items. 324 """ 325 items = [] 326 next_page_token = None 327 while True: 328 api = api_resource(pageToken=next_page_token, *args, **kwargs) 329 response = self.Execute(api) 330 items.extend(response.get("items", [])) 331 next_page_token = response.get("nextPageToken") 332 if not next_page_token: 333 break 334 return items 335 336 @property 337 def service(self): 338 """Return self._service as a property.""" 339 return self._service 340