#!/usr/bin/env python # # Copyright 2016 - The Android Open Source Project # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Base Cloud API Client. BasicCloudApiCliend does basic setup for a cloud API. """ import logging import socket import ssl import six from six.moves import http_client # pylint: disable=import-error import httplib2 from apiclient import errors as gerrors from apiclient.discovery import build from oauth2client import client from acloud import errors from acloud.internal.lib import utils logger = logging.getLogger(__name__) class BaseCloudApiClient(): """A class that does basic setup for a cloud API.""" # To be overriden by subclasses. API_NAME = "" API_VERSION = "v1" SCOPE = "" # Defaults for retry. RETRY_COUNT = 5 RETRY_BACKOFF_FACTOR = 1.5 RETRY_SLEEP_MULTIPLIER = 2 RETRY_HTTP_CODES = [ # 403 is to retry the "Rate Limit Exceeded" error. # We could retry on a finer-grained error message later if necessary. 403, 500, # Internal Server Error 502, # Bad Gateway 503, # Service Unavailable ] RETRIABLE_ERRORS = (http_client.HTTPException, httplib2.HttpLib2Error, socket.error, ssl.SSLError) RETRIABLE_AUTH_ERRORS = (client.AccessTokenRefreshError, ) def __init__(self, oauth2_credentials): """Initialize. Args: oauth2_credentials: An oauth2client.OAuth2Credentials instance. """ self._service = self.InitResourceHandle(oauth2_credentials) @classmethod def InitResourceHandle(cls, oauth2_credentials): """Authenticate and initialize a Resource object. Authenticate http and create a Resource object with methods for interacting with the service. Args: oauth2_credentials: An oauth2client.OAuth2Credentials instance. Returns: An apiclient.discovery.Resource object """ http_auth = oauth2_credentials.authorize(httplib2.Http()) return utils.RetryExceptionType( exception_types=cls.RETRIABLE_AUTH_ERRORS, max_retries=cls.RETRY_COUNT, functor=build, sleep_multiplier=cls.RETRY_SLEEP_MULTIPLIER, retry_backoff_factor=cls.RETRY_BACKOFF_FACTOR, serviceName=cls.API_NAME, version=cls.API_VERSION, # This is workaround for a known issue of some veriosn # of api client. # https://github.com/google/google-api-python-client/issues/435 cache_discovery=False, http=http_auth) @staticmethod def _ShouldRetry(exception, retry_http_codes, other_retriable_errors): """Check if exception is retriable. Args: exception: An instance of Exception. retry_http_codes: a list of integers, retriable HTTP codes of HttpError other_retriable_errors: a tuple of error types to retry other than HttpError. Returns: Boolean, True if retriable, False otherwise. """ if isinstance(exception, other_retriable_errors): return True if isinstance(exception, errors.HttpError): if exception.code in retry_http_codes: return True logger.debug("_ShouldRetry: Exception code %s not in %s: %s", exception.code, retry_http_codes, str(exception)) logger.debug("_ShouldRetry: Exception %s is not one of %s: %s", type(exception), list(other_retriable_errors) + [errors.HttpError], str(exception)) return False @staticmethod def _TranslateError(exception): """Translate the exception to a desired type. Args: exception: An instance of Exception. Returns: gerrors.HttpError will be translated to errors.HttpError. If the error code is errors.HTTP_NOT_FOUND_CODE, it will be translated to errors.ResourceNotFoundError. Unrecognized error type will not be translated and will be returned as is. """ if isinstance(exception, gerrors.HttpError): exception = errors.HttpError.CreateFromHttpError(exception) if exception.code == errors.HTTP_NOT_FOUND_CODE: exception = errors.ResourceNotFoundError( exception.code, str(exception)) return exception def ExecuteOnce(self, api): """Execute an api and parse the errors. Args: api: An apiclient.http.HttpRequest, representing the api to execute. Returns: Execution result of the api. Raises: errors.ResourceNotFoundError: For 404 error. errors.HttpError: For other types of http error. """ try: return api.execute() except gerrors.HttpError as e: raise self._TranslateError(e) def Execute(self, api, retry_http_codes=None, max_retry=None, sleep=None, backoff_factor=None, other_retriable_errors=None): """Execute an api with retry. Call ExecuteOnce and retry on http error with given codes. Args: api: An apiclient.http.HttpRequest, representing the api to execute: retry_http_codes: A list of http codes to retry. max_retry: See utils.Retry. sleep: See utils.Retry. backoff_factor: See utils.Retry. other_retriable_errors: A tuple of error types that should be retried other than errors.HttpError. Returns: Execution result of the api. Raises: See ExecuteOnce. """ retry_http_codes = (self.RETRY_HTTP_CODES if retry_http_codes is None else retry_http_codes) max_retry = (self.RETRY_COUNT if max_retry is None else max_retry) sleep = (self.RETRY_SLEEP_MULTIPLIER if sleep is None else sleep) backoff_factor = (self.RETRY_BACKOFF_FACTOR if backoff_factor is None else backoff_factor) other_retriable_errors = (self.RETRIABLE_ERRORS if other_retriable_errors is None else other_retriable_errors) def _Handler(exc): """Check if |exc| is a retriable exception. Args: exc: An exception. Returns: True if exc is an errors.HttpError and code exists in |retry_http_codes| False otherwise. """ if self._ShouldRetry(exc, retry_http_codes, other_retriable_errors): logger.debug("Will retry error: %s", str(exc)) return True return False return utils.Retry( _Handler, max_retries=max_retry, functor=self.ExecuteOnce, sleep_multiplier=sleep, retry_backoff_factor=backoff_factor, api=api) def BatchExecuteOnce(self, requests): """Execute requests in a batch. Args: requests: A dictionary where key is request id and value is an http request. Returns: results, a dictionary in the following format {request_id: (response, exception)} request_ids are those from requests; response is the http response for the request or None on error; exception is an instance of DriverError or None if no error. """ results = {} def _CallBack(request_id, response, exception): results[request_id] = (response, self._TranslateError(exception)) batch = self._service.new_batch_http_request() for request_id, request in six.iteritems(requests): batch.add( request=request, callback=_CallBack, request_id=request_id) batch.execute() return results def BatchExecute(self, requests, retry_http_codes=None, max_retry=None, sleep=None, backoff_factor=None, other_retriable_errors=None): """Batch execute multiple requests with retry. Call BatchExecuteOnce and retry on http error with given codes. Args: requests: A dictionary where key is request id picked by caller, and value is a apiclient.http.HttpRequest. retry_http_codes: A list of http codes to retry. max_retry: See utils.Retry. sleep: See utils.Retry. backoff_factor: See utils.Retry. other_retriable_errors: A tuple of error types that should be retried other than errors.HttpError. Returns: results, a dictionary in the following format {request_id: (response, exception)} request_ids are those from requests; response is the http response for the request or None on error; exception is an instance of DriverError or None if no error. """ executor = utils.BatchHttpRequestExecutor( self.BatchExecuteOnce, requests=requests, retry_http_codes=retry_http_codes or self.RETRY_HTTP_CODES, max_retry=max_retry or self.RETRY_COUNT, sleep=sleep or self.RETRY_SLEEP_MULTIPLIER, backoff_factor=backoff_factor or self.RETRY_BACKOFF_FACTOR, other_retriable_errors=other_retriable_errors or self.RETRIABLE_ERRORS) executor.Execute() return executor.GetResults() def ListWithMultiPages(self, api_resource, *args, **kwargs): """Call an api that list a type of resource. Multiple google services support listing a type of resource (e.g list gce instances, list storage objects). The querying pattern is similar -- Step 1: execute the api and get a response object like, { "items": [..list of resource..], # The continuation token that can be used # to get the next page. "nextPageToken": "A String", } Step 2: execute the api again with the nextPageToken to retrieve more pages and get a response object. Step 3: Repeat Step 2 until no more page. This method encapsulates the generic logic of calling such listing api. Args: api_resource: An apiclient.discovery.Resource object used to create an http request for the listing api. *args: Arguments used to create the http request. **kwargs: Keyword based arguments to create the http request. Returns: A list of items. """ items = [] next_page_token = None while True: api = api_resource(pageToken=next_page_token, *args, **kwargs) response = self.Execute(api) items.extend(response.get("items", [])) next_page_token = response.get("nextPageToken") if not next_page_token: break return items @property def service(self): """Return self._service as a property.""" return self._service