1#!/usr/bin/env python 2# 3# Copyright 2016 - The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16"""Base Cloud API Client. 17 18BasicCloudApiCliend does basic setup for a cloud API. 19""" 20import logging 21import socket 22import ssl 23 24import six 25from six.moves import http_client 26 27# pylint: disable=import-error 28import httplib2 29from apiclient import errors as gerrors 30from apiclient.discovery import build 31from oauth2client import client 32 33from acloud import errors 34from acloud.internal.lib import utils 35 36 37logger = logging.getLogger(__name__) 38 39 40class BaseCloudApiClient(): 41 """A class that does basic setup for a cloud API.""" 42 43 # To be overriden by subclasses. 44 API_NAME = "" 45 API_VERSION = "v1" 46 SCOPE = "" 47 48 # Defaults for retry. 49 RETRY_COUNT = 5 50 RETRY_BACKOFF_FACTOR = 1.5 51 RETRY_SLEEP_MULTIPLIER = 2 52 RETRY_HTTP_CODES = [ 53 # 403 is to retry the "Rate Limit Exceeded" error. 54 # We could retry on a finer-grained error message later if necessary. 55 403, 56 500, # Internal Server Error 57 502, # Bad Gateway 58 503, # Service Unavailable 59 ] 60 RETRIABLE_ERRORS = (http_client.HTTPException, httplib2.HttpLib2Error, 61 socket.error, ssl.SSLError) 62 RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) 63 64 def __init__(self, oauth2_credentials): 65 """Initialize. 66 67 Args: 68 oauth2_credentials: An oauth2client.OAuth2Credentials instance. 69 """ 70 self._service = self.InitResourceHandle(oauth2_credentials) 71 72 @classmethod 73 def InitResourceHandle(cls, oauth2_credentials): 74 """Authenticate and initialize a Resource object. 75 76 Authenticate http and create a Resource object with methods 77 for interacting with the service. 78 79 Args: 80 oauth2_credentials: An oauth2client.OAuth2Credentials instance. 81 82 Returns: 83 An apiclient.discovery.Resource object 84 """ 85 http_auth = oauth2_credentials.authorize(httplib2.Http()) 86 return utils.RetryExceptionType( 87 exception_types=cls.RETRIABLE_AUTH_ERRORS, 88 max_retries=cls.RETRY_COUNT, 89 functor=build, 90 sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER, 91 retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR, 92 serviceName=cls.API_NAME, 93 version=cls.API_VERSION, 94 # This is workaround for a known issue of some veriosn 95 # of api client. 96 # https://github.com/google/google-api-python-client/issues/435 97 cache_discovery=False, 98 http=http_auth) 99 100 @staticmethod 101 def _ShouldRetry(exception, retry_http_codes, 102 other_retriable_errors): 103 """Check if exception is retriable. 104 105 Args: 106 exception: An instance of Exception. 107 retry_http_codes: a list of integers, retriable HTTP codes of 108 HttpError 109 other_retriable_errors: a tuple of error types to retry other than 110 HttpError. 111 112 Returns: 113 Boolean, True if retriable, False otherwise. 114 """ 115 if isinstance(exception, other_retriable_errors): 116 return True 117 118 if isinstance(exception, errors.HttpError): 119 if exception.code in retry_http_codes: 120 return True 121 logger.debug("_ShouldRetry: Exception code %s not in %s: %s", 122 exception.code, retry_http_codes, str(exception)) 123 124 logger.debug("_ShouldRetry: Exception %s is not one of %s: %s", 125 type(exception), 126 list(other_retriable_errors) + [errors.HttpError], 127 str(exception)) 128 return False 129 130 @staticmethod 131 def _TranslateError(exception): 132 """Translate the exception to a desired type. 133 134 Args: 135 exception: An instance of Exception. 136 137 Returns: 138 gerrors.HttpError will be translated to errors.HttpError. 139 If the error code is errors.HTTP_NOT_FOUND_CODE, it will 140 be translated to errors.ResourceNotFoundError. 141 Unrecognized error type will not be translated and will 142 be returned as is. 143 """ 144 if isinstance(exception, gerrors.HttpError): 145 exception = errors.HttpError.CreateFromHttpError(exception) 146 if exception.code == errors.HTTP_NOT_FOUND_CODE: 147 exception = errors.ResourceNotFoundError( 148 exception.code, str(exception)) 149 return exception 150 151 def ExecuteOnce(self, api): 152 """Execute an api and parse the errors. 153 154 Args: 155 api: An apiclient.http.HttpRequest, representing the api to execute. 156 157 Returns: 158 Execution result of the api. 159 160 Raises: 161 errors.ResourceNotFoundError: For 404 error. 162 errors.HttpError: For other types of http error. 163 """ 164 try: 165 return api.execute() 166 except gerrors.HttpError as e: 167 raise self._TranslateError(e) 168 169 def Execute(self, 170 api, 171 retry_http_codes=None, 172 max_retry=None, 173 sleep=None, 174 backoff_factor=None, 175 other_retriable_errors=None): 176 """Execute an api with retry. 177 178 Call ExecuteOnce and retry on http error with given codes. 179 180 Args: 181 api: An apiclient.http.HttpRequest, representing the api to execute: 182 retry_http_codes: A list of http codes to retry. 183 max_retry: See utils.Retry. 184 sleep: See utils.Retry. 185 backoff_factor: See utils.Retry. 186 other_retriable_errors: A tuple of error types that should be retried 187 other than errors.HttpError. 188 189 Returns: 190 Execution result of the api. 191 192 Raises: 193 See ExecuteOnce. 194 """ 195 retry_http_codes = (self.RETRY_HTTP_CODES 196 if retry_http_codes is None else retry_http_codes) 197 max_retry = (self.RETRY_COUNT if max_retry is None else max_retry) 198 sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep) 199 backoff_factor = (self.RETRY_BACKOFF_FACTOR 200 if backoff_factor is None else backoff_factor) 201 other_retriable_errors = (self.RETRIABLE_ERRORS 202 if other_retriable_errors is None else 203 other_retriable_errors) 204 205 def _Handler(exc): 206 """Check if |exc| is a retriable exception. 207 208 Args: 209 exc: An exception. 210 211 Returns: 212 True if exc is an errors.HttpError and code exists in |retry_http_codes| 213 False otherwise. 214 """ 215 if self._ShouldRetry(exc, retry_http_codes, 216 other_retriable_errors): 217 logger.debug("Will retry error: %s", str(exc)) 218 return True 219 return False 220 221 return utils.Retry( 222 _Handler, 223 max_retries=max_retry, 224 functor=self.ExecuteOnce, 225 sleep_multiplier=sleep, 226 retry_backoff_factor=backoff_factor, 227 api=api) 228 229 def BatchExecuteOnce(self, requests): 230 """Execute requests in a batch. 231 232 Args: 233 requests: A dictionary where key is request id and value 234 is an http request. 235 236 Returns: 237 results, a dictionary in the following format 238 {request_id: (response, exception)} 239 request_ids are those from requests; response 240 is the http response for the request or None on error; 241 exception is an instance of DriverError or None if no error. 242 """ 243 results = {} 244 245 def _CallBack(request_id, response, exception): 246 results[request_id] = (response, self._TranslateError(exception)) 247 248 batch = self._service.new_batch_http_request() 249 for request_id, request in six.iteritems(requests): 250 batch.add( 251 request=request, callback=_CallBack, request_id=request_id) 252 batch.execute() 253 return results 254 255 def BatchExecute(self, 256 requests, 257 retry_http_codes=None, 258 max_retry=None, 259 sleep=None, 260 backoff_factor=None, 261 other_retriable_errors=None): 262 """Batch execute multiple requests with retry. 263 264 Call BatchExecuteOnce and retry on http error with given codes. 265 266 Args: 267 requests: A dictionary where key is request id picked by caller, 268 and value is a apiclient.http.HttpRequest. 269 retry_http_codes: A list of http codes to retry. 270 max_retry: See utils.Retry. 271 sleep: See utils.Retry. 272 backoff_factor: See utils.Retry. 273 other_retriable_errors: A tuple of error types that should be retried 274 other than errors.HttpError. 275 276 Returns: 277 results, a dictionary in the following format 278 {request_id: (response, exception)} 279 request_ids are those from requests; response 280 is the http response for the request or None on error; 281 exception is an instance of DriverError or None if no error. 282 """ 283 executor = utils.BatchHttpRequestExecutor( 284 self.BatchExecuteOnce, 285 requests=requests, 286 retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES, 287 max_retry=max_retry or self.RETRY_COUNT, 288 sleep=sleep or self.RETRY_SLEEP_MULTIPLIER, 289 backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR, 290 other_retriable_errors=other_retriable_errors 291 or self.RETRIABLE_ERRORS) 292 executor.Execute() 293 return executor.GetResults() 294 295 def ListWithMultiPages(self, api_resource, *args, **kwargs): 296 """Call an api that list a type of resource. 297 298 Multiple google services support listing a type of 299 resource (e.g list gce instances, list storage objects). 300 The querying pattern is similar -- 301 Step 1: execute the api and get a response object like, 302 { 303 "items": [..list of resource..], 304 # The continuation token that can be used 305 # to get the next page. 306 "nextPageToken": "A String", 307 } 308 Step 2: execute the api again with the nextPageToken to 309 retrieve more pages and get a response object. 310 311 Step 3: Repeat Step 2 until no more page. 312 313 This method encapsulates the generic logic of 314 calling such listing api. 315 316 Args: 317 api_resource: An apiclient.discovery.Resource object 318 used to create an http request for the listing api. 319 *args: Arguments used to create the http request. 320 **kwargs: Keyword based arguments to create the http 321 request. 322 323 Returns: 324 A list of items. 325 """ 326 items = [] 327 next_page_token = None 328 while True: 329 api = api_resource(pageToken=next_page_token, *args, **kwargs) 330 response = self.Execute(api) 331 items.extend(response.get("items", [])) 332 next_page_token = response.get("nextPageToken") 333 if not next_page_token: 334 break 335 return items 336 337 @property 338 def service(self): 339 """Return self._service as a property.""" 340 return self._service 341