1#!/usr/bin/env python 2# 3# Copyright 2015 Google Inc. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Common credentials classes and constructors.""" 18from __future__ import print_function 19 20import datetime 21import json 22import os 23import threading 24import warnings 25 26import httplib2 27import oauth2client 28import oauth2client.client 29from oauth2client import service_account 30from oauth2client import tools # for gflags declarations 31from six.moves import http_client 32from six.moves import urllib 33 34from apitools.base.py import exceptions 35from apitools.base.py import util 36 37# Note: we try the oauth2client imports two ways, to accomodate layout 38# changes in oauth2client 2.0+. We can remove these once we no longer 39# support oauth2client < 2.0. 40# 41# pylint: disable=wrong-import-order,ungrouped-imports 42try: 43 from oauth2client.contrib import gce 44except ImportError: 45 from oauth2client import gce 46 47try: 48 from oauth2client.contrib import locked_file 49except ImportError: 50 from oauth2client import locked_file 51 52try: 53 from oauth2client.contrib import multistore_file 54except ImportError: 55 from oauth2client import multistore_file 56 57try: 58 import gflags 59 FLAGS = gflags.FLAGS 60except ImportError: 61 FLAGS = None 62 63 64__all__ = [ 65 'CredentialsFromFile', 66 'GaeAssertionCredentials', 67 'GceAssertionCredentials', 68 'GetCredentials', 69 'GetUserinfo', 70 'ServiceAccountCredentialsFromFile', 71] 72 73 74# Lock when accessing the cache file to avoid resource contention. 75cache_file_lock = threading.Lock() 76 77 78def SetCredentialsCacheFileLock(lock): 79 global cache_file_lock # pylint: disable=global-statement 80 cache_file_lock = lock 81 82 83# List of additional methods we use when attempting to construct 84# credentials. Users can register their own methods here, which we try 85# before the defaults. 86_CREDENTIALS_METHODS = [] 87 88 89def _RegisterCredentialsMethod(method, position=None): 90 """Register a new method for fetching credentials. 91 92 This new method should be a function with signature: 93 client_info, **kwds -> Credentials or None 94 This method can be used as a decorator, unless position needs to 95 be supplied. 96 97 Note that method must *always* accept arbitrary keyword arguments. 98 99 Args: 100 method: New credential-fetching method. 101 position: (default: None) Where in the list of methods to 102 add this; if None, we append. In all but rare cases, 103 this should be either 0 or None. 104 Returns: 105 method, for use as a decorator. 106 107 """ 108 if position is None: 109 position = len(_CREDENTIALS_METHODS) 110 else: 111 position = min(position, len(_CREDENTIALS_METHODS)) 112 _CREDENTIALS_METHODS.insert(position, method) 113 return method 114 115 116def GetCredentials(package_name, scopes, client_id, client_secret, user_agent, 117 credentials_filename=None, 118 api_key=None, # pylint: disable=unused-argument 119 client=None, # pylint: disable=unused-argument 120 oauth2client_args=None, 121 **kwds): 122 """Attempt to get credentials, using an oauth dance as the last resort.""" 123 scopes = util.NormalizeScopes(scopes) 124 client_info = { 125 'client_id': client_id, 126 'client_secret': client_secret, 127 'scope': ' '.join(sorted(scopes)), 128 'user_agent': user_agent or '%s-generated/0.1' % package_name, 129 } 130 for method in _CREDENTIALS_METHODS: 131 credentials = method(client_info, **kwds) 132 if credentials is not None: 133 return credentials 134 credentials_filename = credentials_filename or os.path.expanduser( 135 '~/.apitools.token') 136 credentials = CredentialsFromFile(credentials_filename, client_info, 137 oauth2client_args=oauth2client_args) 138 if credentials is not None: 139 return credentials 140 raise exceptions.CredentialsError('Could not create valid credentials') 141 142 143def ServiceAccountCredentialsFromFile(filename, scopes, user_agent=None): 144 """Use the credentials in filename to create a token for scopes.""" 145 filename = os.path.expanduser(filename) 146 # We have two options, based on our version of oauth2client. 147 if oauth2client.__version__ > '1.5.2': 148 # oauth2client >= 2.0.0 149 credentials = ( 150 service_account.ServiceAccountCredentials.from_json_keyfile_name( 151 filename, scopes=scopes)) 152 if credentials is not None: 153 if user_agent is not None: 154 credentials.user_agent = user_agent 155 return credentials 156 else: 157 # oauth2client < 2.0.0 158 with open(filename) as keyfile: 159 service_account_info = json.load(keyfile) 160 account_type = service_account_info.get('type') 161 if account_type != oauth2client.client.SERVICE_ACCOUNT: 162 raise exceptions.CredentialsError( 163 'Invalid service account credentials: %s' % (filename,)) 164 # pylint: disable=protected-access 165 credentials = service_account._ServiceAccountCredentials( 166 service_account_id=service_account_info['client_id'], 167 service_account_email=service_account_info['client_email'], 168 private_key_id=service_account_info['private_key_id'], 169 private_key_pkcs8_text=service_account_info['private_key'], 170 scopes=scopes, user_agent=user_agent) 171 # pylint: enable=protected-access 172 return credentials 173 174 175def ServiceAccountCredentialsFromP12File( 176 service_account_name, private_key_filename, scopes, user_agent): 177 """Create a new credential from the named .p12 keyfile.""" 178 private_key_filename = os.path.expanduser(private_key_filename) 179 scopes = util.NormalizeScopes(scopes) 180 if oauth2client.__version__ > '1.5.2': 181 # oauth2client >= 2.0.0 182 credentials = ( 183 service_account.ServiceAccountCredentials.from_p12_keyfile( 184 service_account_name, private_key_filename, scopes=scopes)) 185 if credentials is not None: 186 credentials.user_agent = user_agent 187 return credentials 188 else: 189 # oauth2client < 2.0.0 190 with open(private_key_filename) as key_file: 191 return oauth2client.client.SignedJwtAssertionCredentials( 192 service_account_name, key_file.read(), scopes, 193 user_agent=user_agent) 194 195 196def _EnsureFileExists(filename): 197 """Touches a file; returns False on error, True on success.""" 198 if not os.path.exists(filename): 199 old_umask = os.umask(0o177) 200 try: 201 open(filename, 'a+b').close() 202 except OSError: 203 return False 204 finally: 205 os.umask(old_umask) 206 return True 207 208 209def _GceMetadataRequest(relative_url, use_metadata_ip=False): 210 """Request the given url from the GCE metadata service.""" 211 if use_metadata_ip: 212 base_url = os.environ.get('GCE_METADATA_IP', '169.254.169.254') 213 else: 214 base_url = os.environ.get( 215 'GCE_METADATA_ROOT', 'metadata.google.internal') 216 url = 'http://' + base_url + '/computeMetadata/v1/' + relative_url 217 # Extra header requirement can be found here: 218 # https://developers.google.com/compute/docs/metadata 219 headers = {'Metadata-Flavor': 'Google'} 220 request = urllib.request.Request(url, headers=headers) 221 opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) 222 try: 223 response = opener.open(request) 224 except urllib.error.URLError as e: 225 raise exceptions.CommunicationError( 226 'Could not reach metadata service: %s' % e.reason) 227 return response 228 229 230class GceAssertionCredentials(gce.AppAssertionCredentials): 231 232 """Assertion credentials for GCE instances.""" 233 234 def __init__(self, scopes=None, service_account_name='default', **kwds): 235 """Initializes the credentials instance. 236 237 Args: 238 scopes: The scopes to get. If None, whatever scopes that are 239 available to the instance are used. 240 service_account_name: The service account to retrieve the scopes 241 from. 242 **kwds: Additional keyword args. 243 244 """ 245 # If there is a connectivity issue with the metadata server, 246 # detection calls may fail even if we've already successfully 247 # identified these scopes in the same execution. However, the 248 # available scopes don't change once an instance is created, 249 # so there is no reason to perform more than one query. 250 self.__service_account_name = service_account_name 251 cached_scopes = None 252 cache_filename = kwds.get('cache_filename') 253 if cache_filename: 254 cached_scopes = self._CheckCacheFileForMatch( 255 cache_filename, scopes) 256 257 scopes = cached_scopes or self._ScopesFromMetadataServer(scopes) 258 259 if cache_filename and not cached_scopes: 260 self._WriteCacheFile(cache_filename, scopes) 261 262 # We check the scopes above, but don't need them again after 263 # this point. Newer versions of oauth2client let us drop them 264 # here, but since we support older versions as well, we just 265 # catch and squelch the warning. 266 with warnings.catch_warnings(): 267 warnings.simplefilter('ignore') 268 super(GceAssertionCredentials, self).__init__(scopes, **kwds) 269 270 @classmethod 271 def Get(cls, *args, **kwds): 272 try: 273 return cls(*args, **kwds) 274 except exceptions.Error: 275 return None 276 277 def _CheckCacheFileForMatch(self, cache_filename, scopes): 278 """Checks the cache file to see if it matches the given credentials. 279 280 Args: 281 cache_filename: Cache filename to check. 282 scopes: Scopes for the desired credentials. 283 284 Returns: 285 List of scopes (if cache matches) or None. 286 """ 287 creds = { # Credentials metadata dict. 288 'scopes': sorted(list(scopes)) if scopes else None, 289 'svc_acct_name': self.__service_account_name, 290 } 291 with cache_file_lock: 292 if _EnsureFileExists(cache_filename): 293 cache_file = locked_file.LockedFile( 294 cache_filename, 'r+b', 'rb') 295 try: 296 cache_file.open_and_lock() 297 cached_creds_str = cache_file.file_handle().read() 298 if cached_creds_str: 299 # Cached credentials metadata dict. 300 cached_creds = json.loads(cached_creds_str) 301 if (creds['svc_acct_name'] == 302 cached_creds['svc_acct_name']): 303 if (creds['scopes'] in 304 (None, cached_creds['scopes'])): 305 scopes = cached_creds['scopes'] 306 except KeyboardInterrupt: 307 raise 308 except: # pylint: disable=bare-except 309 # Treat exceptions as a cache miss. 310 pass 311 finally: 312 cache_file.unlock_and_close() 313 return scopes 314 315 def _WriteCacheFile(self, cache_filename, scopes): 316 """Writes the credential metadata to the cache file. 317 318 This does not save the credentials themselves (CredentialStore class 319 optionally handles that after this class is initialized). 320 321 Args: 322 cache_filename: Cache filename to check. 323 scopes: Scopes for the desired credentials. 324 """ 325 with cache_file_lock: 326 if _EnsureFileExists(cache_filename): 327 cache_file = locked_file.LockedFile( 328 cache_filename, 'r+b', 'rb') 329 try: 330 cache_file.open_and_lock() 331 if cache_file.is_locked(): 332 creds = { # Credentials metadata dict. 333 'scopes': sorted(list(scopes)), 334 'svc_acct_name': self.__service_account_name} 335 cache_file.file_handle().write( 336 json.dumps(creds, encoding='ascii')) 337 # If it's not locked, the locking process will 338 # write the same data to the file, so just 339 # continue. 340 except KeyboardInterrupt: 341 raise 342 except: # pylint: disable=bare-except 343 # Treat exceptions as a cache miss. 344 pass 345 finally: 346 cache_file.unlock_and_close() 347 348 def _ScopesFromMetadataServer(self, scopes): 349 """Returns instance scopes based on GCE metadata server.""" 350 if not util.DetectGce(): 351 raise exceptions.ResourceUnavailableError( 352 'GCE credentials requested outside a GCE instance') 353 if not self.GetServiceAccount(self.__service_account_name): 354 raise exceptions.ResourceUnavailableError( 355 'GCE credentials requested but service account ' 356 '%s does not exist.' % self.__service_account_name) 357 if scopes: 358 scope_ls = util.NormalizeScopes(scopes) 359 instance_scopes = self.GetInstanceScopes() 360 if scope_ls > instance_scopes: 361 raise exceptions.CredentialsError( 362 'Instance did not have access to scopes %s' % ( 363 sorted(list(scope_ls - instance_scopes)),)) 364 else: 365 scopes = self.GetInstanceScopes() 366 return scopes 367 368 def GetServiceAccount(self, account): 369 relative_url = 'instance/service-accounts' 370 response = _GceMetadataRequest(relative_url) 371 response_lines = [line.rstrip('/\n\r') 372 for line in response.readlines()] 373 return account in response_lines 374 375 def GetInstanceScopes(self): 376 relative_url = 'instance/service-accounts/{0}/scopes'.format( 377 self.__service_account_name) 378 response = _GceMetadataRequest(relative_url) 379 return util.NormalizeScopes(scope.strip() 380 for scope in response.readlines()) 381 382 # pylint: disable=arguments-differ 383 def _refresh(self, do_request): 384 """Refresh self.access_token. 385 386 This function replaces AppAssertionCredentials._refresh, which 387 does not use the credential store and is therefore poorly 388 suited for multi-threaded scenarios. 389 390 Args: 391 do_request: A function matching httplib2.Http.request's signature. 392 393 """ 394 # pylint: disable=protected-access 395 oauth2client.client.OAuth2Credentials._refresh(self, do_request) 396 # pylint: enable=protected-access 397 398 def _do_refresh_request(self, unused_http_request): 399 """Refresh self.access_token by querying the metadata server. 400 401 If self.store is initialized, store acquired credentials there. 402 """ 403 relative_url = 'instance/service-accounts/{0}/token'.format( 404 self.__service_account_name) 405 try: 406 response = _GceMetadataRequest(relative_url) 407 except exceptions.CommunicationError: 408 self.invalid = True 409 if self.store: 410 self.store.locked_put(self) 411 raise 412 content = response.read() 413 try: 414 credential_info = json.loads(content) 415 except ValueError: 416 raise exceptions.CredentialsError( 417 'Could not parse response as JSON: %s' % content) 418 419 self.access_token = credential_info['access_token'] 420 if 'expires_in' in credential_info: 421 expires_in = int(credential_info['expires_in']) 422 self.token_expiry = ( 423 datetime.timedelta(seconds=expires_in) + 424 datetime.datetime.utcnow()) 425 else: 426 self.token_expiry = None 427 self.invalid = False 428 if self.store: 429 self.store.locked_put(self) 430 431 @classmethod 432 def from_json(cls, json_data): 433 data = json.loads(json_data) 434 kwargs = {} 435 if 'cache_filename' in data.get('kwargs', []): 436 kwargs['cache_filename'] = data['kwargs']['cache_filename'] 437 credentials = GceAssertionCredentials(scopes=[data['scope']], 438 **kwargs) 439 if 'access_token' in data: 440 credentials.access_token = data['access_token'] 441 if 'token_expiry' in data: 442 credentials.token_expiry = datetime.datetime.strptime( 443 data['token_expiry'], oauth2client.client.EXPIRY_FORMAT) 444 if 'invalid' in data: 445 credentials.invalid = data['invalid'] 446 return credentials 447 448 @property 449 def serialization_data(self): 450 raise NotImplementedError( 451 'Cannot serialize credentials for GCE service accounts.') 452 453 454# TODO(craigcitro): Currently, we can't even *load* 455# `oauth2client.appengine` without being on appengine, because of how 456# it handles imports. Fix that by splitting that module into 457# GAE-specific and GAE-independent bits, and guarding imports. 458class GaeAssertionCredentials(oauth2client.client.AssertionCredentials): 459 460 """Assertion credentials for Google App Engine apps.""" 461 462 def __init__(self, scopes, **kwds): 463 if not util.DetectGae(): 464 raise exceptions.ResourceUnavailableError( 465 'GCE credentials requested outside a GCE instance') 466 self._scopes = list(util.NormalizeScopes(scopes)) 467 super(GaeAssertionCredentials, self).__init__(None, **kwds) 468 469 @classmethod 470 def Get(cls, *args, **kwds): 471 try: 472 return cls(*args, **kwds) 473 except exceptions.Error: 474 return None 475 476 @classmethod 477 def from_json(cls, json_data): 478 data = json.loads(json_data) 479 return GaeAssertionCredentials(data['_scopes']) 480 481 def _refresh(self, _): 482 """Refresh self.access_token. 483 484 Args: 485 _: (ignored) A function matching httplib2.Http.request's signature. 486 """ 487 # pylint: disable=import-error 488 from google.appengine.api import app_identity 489 try: 490 token, _ = app_identity.get_access_token(self._scopes) 491 except app_identity.Error as e: 492 raise exceptions.CredentialsError(str(e)) 493 self.access_token = token 494 495 def sign_blob(self, blob): 496 """Cryptographically sign a blob (of bytes). 497 498 This method is provided to support a common interface, but 499 the actual key used for a Google Compute Engine service account 500 is not available, so it can't be used to sign content. 501 502 Args: 503 blob: bytes, Message to be signed. 504 505 Raises: 506 NotImplementedError, always. 507 """ 508 raise NotImplementedError( 509 'Compute Engine service accounts cannot sign blobs') 510 511 512def _GetRunFlowFlags(args=None): 513 """Retrieves command line flags based on gflags module.""" 514 # There's one rare situation where gsutil will not have argparse 515 # available, but doesn't need anything depending on argparse anyway, 516 # since they're bringing their own credentials. So we just allow this 517 # to fail with an ImportError in those cases. 518 # 519 # TODO(craigcitro): Move this import back to the top when we drop 520 # python 2.6 support (eg when gsutil does). 521 import argparse 522 523 parser = argparse.ArgumentParser(parents=[tools.argparser]) 524 # Get command line argparse flags. 525 flags, _ = parser.parse_known_args(args=args) 526 527 # Allow `gflags` and `argparse` to be used side-by-side. 528 if hasattr(FLAGS, 'auth_host_name'): 529 flags.auth_host_name = FLAGS.auth_host_name 530 if hasattr(FLAGS, 'auth_host_port'): 531 flags.auth_host_port = FLAGS.auth_host_port 532 if hasattr(FLAGS, 'auth_local_webserver'): 533 flags.noauth_local_webserver = (not FLAGS.auth_local_webserver) 534 return flags 535 536 537# TODO(craigcitro): Switch this from taking a path to taking a stream. 538def CredentialsFromFile(path, client_info, oauth2client_args=None): 539 """Read credentials from a file.""" 540 credential_store = multistore_file.get_credential_storage( 541 path, 542 client_info['client_id'], 543 client_info['user_agent'], 544 client_info['scope']) 545 if hasattr(FLAGS, 'auth_local_webserver'): 546 FLAGS.auth_local_webserver = False 547 credentials = credential_store.get() 548 if credentials is None or credentials.invalid: 549 print('Generating new OAuth credentials ...') 550 for _ in range(20): 551 # If authorization fails, we want to retry, rather than let this 552 # cascade up and get caught elsewhere. If users want out of the 553 # retry loop, they can ^C. 554 try: 555 flow = oauth2client.client.OAuth2WebServerFlow(**client_info) 556 flags = _GetRunFlowFlags(args=oauth2client_args) 557 credentials = tools.run_flow(flow, credential_store, flags) 558 break 559 except (oauth2client.client.FlowExchangeError, SystemExit) as e: 560 # Here SystemExit is "no credential at all", and the 561 # FlowExchangeError is "invalid" -- usually because 562 # you reused a token. 563 print('Invalid authorization: %s' % (e,)) 564 except httplib2.HttpLib2Error as e: 565 print('Communication error: %s' % (e,)) 566 raise exceptions.CredentialsError( 567 'Communication error creating credentials: %s' % e) 568 return credentials 569 570 571# TODO(craigcitro): Push this into oauth2client. 572def GetUserinfo(credentials, http=None): # pylint: disable=invalid-name 573 """Get the userinfo associated with the given credentials. 574 575 This is dependent on the token having either the userinfo.email or 576 userinfo.profile scope for the given token. 577 578 Args: 579 credentials: (oauth2client.client.Credentials) incoming credentials 580 http: (httplib2.Http, optional) http instance to use 581 582 Returns: 583 The email address for this token, or None if the required scopes 584 aren't available. 585 """ 586 http = http or httplib2.Http() 587 url = _GetUserinfoUrl(credentials) 588 # We ignore communication woes here (i.e. SSL errors, socket 589 # timeout), as handling these should be done in a common location. 590 response, content = http.request(url) 591 if response.status == http_client.BAD_REQUEST: 592 credentials.refresh(http) 593 url = _GetUserinfoUrl(credentials) 594 response, content = http.request(url) 595 return json.loads(content or '{}') # Save ourselves from an empty reply. 596 597 598def _GetUserinfoUrl(credentials): 599 url_root = 'https://www.googleapis.com/oauth2/v2/tokeninfo' 600 query_args = {'access_token': credentials.access_token} 601 return '?'.join((url_root, urllib.parse.urlencode(query_args))) 602 603 604@_RegisterCredentialsMethod 605def _GetServiceAccountCredentials( 606 client_info, service_account_name=None, service_account_keyfile=None, 607 service_account_json_keyfile=None, **unused_kwds): 608 """Returns ServiceAccountCredentials from give file.""" 609 if ((service_account_name and not service_account_keyfile) or 610 (service_account_keyfile and not service_account_name)): 611 raise exceptions.CredentialsError( 612 'Service account name or keyfile provided without the other') 613 scopes = client_info['scope'].split() 614 user_agent = client_info['user_agent'] 615 # Use the .json credentials, if provided. 616 if service_account_json_keyfile: 617 return ServiceAccountCredentialsFromFile( 618 service_account_json_keyfile, scopes, user_agent=user_agent) 619 # Fall back to .p12 if there's no .json credentials. 620 if service_account_name is not None: 621 return ServiceAccountCredentialsFromP12File( 622 service_account_name, service_account_keyfile, scopes, user_agent) 623 624 625@_RegisterCredentialsMethod 626def _GetGaeServiceAccount(client_info, **unused_kwds): 627 scopes = client_info['scope'].split(' ') 628 return GaeAssertionCredentials.Get(scopes=scopes) 629 630 631@_RegisterCredentialsMethod 632def _GetGceServiceAccount(client_info, **unused_kwds): 633 scopes = client_info['scope'].split(' ') 634 return GceAssertionCredentials.Get(scopes=scopes) 635 636 637@_RegisterCredentialsMethod 638def _GetApplicationDefaultCredentials( 639 client_info, skip_application_default_credentials=False, 640 **unused_kwds): 641 """Returns ADC with right scopes.""" 642 scopes = client_info['scope'].split() 643 if skip_application_default_credentials: 644 return None 645 gc = oauth2client.client.GoogleCredentials 646 with cache_file_lock: 647 try: 648 # pylint: disable=protected-access 649 # We've already done our own check for GAE/GCE 650 # credentials, we don't want to pay for checking again. 651 credentials = gc._implicit_credentials_from_files() 652 except oauth2client.client.ApplicationDefaultCredentialsError: 653 return None 654 # If we got back a non-service account credential, we need to use 655 # a heuristic to decide whether or not the application default 656 # credential will work for us. We assume that if we're requesting 657 # cloud-platform, our scopes are a subset of cloud scopes, and the 658 # ADC will work. 659 cp = 'https://www.googleapis.com/auth/cloud-platform' 660 if credentials is None: 661 return None 662 if not isinstance(credentials, gc) or cp in scopes: 663 return credentials.create_scoped(scopes) 664 return None 665