googleapiclient.http

1 # Copyright 2014 Google Inc. All Rights Reserved. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 """Classes to encapsulate a single HTTP request. 16 17 The classes implement a command pattern, with every 18 object supporting an execute() method that does the 19 actual HTTP request. 20 """ 21 from __future__ import absolute_import 22 import six 23 from six.moves import http_client 24 from six.moves import range 25 26 __author__ = 'jcgregorio@google.com (Joe Gregorio)' 27 28 from six import BytesIO, StringIO 29 from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 30 31 import base64 32 import copy 33 import gzip 34 import httplib2 35 import json 36 import logging 37 import mimetypes 38 import os 39 import random 40 import socket 41 import sys 42 import time 43 import uuid 44 45 # TODO(issue 221): Remove this conditional import jibbajabba. 46 try: 47 import ssl 48 except ImportError: 49 _ssl_SSLError = object() 50 else: 51 _ssl_SSLError = ssl.SSLError 52 53 from email.generator import Generator 54 from email.mime.multipart import MIMEMultipart 55 from email.mime.nonmultipart import MIMENonMultipart 56 from email.parser import FeedParser 57 58 from googleapiclient import _helpers as util 59 60 from googleapiclient import _auth 61 from googleapiclient.errors import BatchError 62 from googleapiclient.errors import HttpError 63 from googleapiclient.errors import InvalidChunkSizeError 64 from googleapiclient.errors import ResumableUploadError 65 from googleapiclient.errors import UnexpectedBodyError 66 from googleapiclient.errors import UnexpectedMethodError 67 from googleapiclient.model import JsonModel 68 69 70 LOGGER = logging.getLogger(__name__) 71 72 DEFAULT_CHUNK_SIZE = 100*1024*1024 73 74 MAX_URI_LENGTH = 2048 75 76 MAX_BATCH_LIMIT = 1000 77 78 _TOO_MANY_REQUESTS = 429 79 80 DEFAULT_HTTP_TIMEOUT_SEC = 60 81 82 _LEGACY_BATCH_URI = 'https://www.googleapis.com/batch'

83 84 85 -def _should_retry_response(resp_status, content):

86 """Determines whether a response should be retried. 87 88 Args: 89 resp_status: The response status received. 90 content: The response content body. 91 92 Returns: 93 True if the response should be retried, otherwise False. 94 """ 95 # Retry on 5xx errors. 96 if resp_status >= 500: 97 return True 98 99 # Retry on 429 errors. 100 if resp_status == _TOO_MANY_REQUESTS: 101 return True 102 103 # For 403 errors, we have to check for the `reason` in the response to 104 # determine if we should retry. 105 if resp_status == six.moves.http_client.FORBIDDEN: 106 # If there's no details about the 403 type, don't retry. 107 if not content: 108 return False 109 110 # Content is in JSON format. 111 try: 112 data = json.loads(content.decode('utf-8')) 113 if isinstance(data, dict): 114 reason = data['error']['errors'][0]['reason'] 115 else: 116 reason = data[0]['error']['errors']['reason'] 117 except (UnicodeDecodeError, ValueError, KeyError): 118 LOGGER.warning('Invalid JSON content from response: %s', content) 119 return False 120 121 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 122 123 # Only retry on rate limit related failures. 124 if reason in ('userRateLimitExceeded', 'rateLimitExceeded', ): 125 return True 126 127 # Everything else is a success or non-retriable so break. 128 return False

129

130 131 -def _retry_request(http, num_retries, req_type, sleep, rand, uri, method, *args, 132 **kwargs):

133 """Retries an HTTP request multiple times while handling errors. 134 135 If after all retries the request still fails, last error is either returned as 136 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 137 138 Args: 139 http: Http object to be used to execute request. 140 num_retries: Maximum number of retries. 141 req_type: Type of the request (used for logging retries). 142 sleep, rand: Functions to sleep for random time between retries. 143 uri: URI to be requested. 144 method: HTTP method to be used. 145 args, kwargs: Additional arguments passed to http.request. 146 147 Returns: 148 resp, content - Response from the http request (may be HTTP 5xx). 149 """ 150 resp = None 151 content = None 152 for retry_num in range(num_retries + 1): 153 if retry_num > 0: 154 # Sleep before retrying. 155 sleep_time = rand() * 2 ** retry_num 156 LOGGER.warning( 157 'Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s', 158 sleep_time, retry_num, num_retries, req_type, method, uri, 159 resp.status if resp else exception) 160 sleep(sleep_time) 161 162 try: 163 exception = None 164 resp, content = http.request(uri, method, *args, **kwargs) 165 # Retry on SSL errors and socket timeout errors. 166 except _ssl_SSLError as ssl_error: 167 exception = ssl_error 168 except socket.timeout as socket_timeout: 169 # It's important that this be before socket.error as it's a subclass 170 # socket.timeout has no errorcode 171 exception = socket_timeout 172 except socket.error as socket_error: 173 # errno's contents differ by platform, so we have to match by name. 174 if socket.errno.errorcode.get(socket_error.errno) not in { 175 'WSAETIMEDOUT', 'ETIMEDOUT', 'EPIPE', 'ECONNABORTED'}: 176 raise 177 exception = socket_error 178 except httplib2.ServerNotFoundError as server_not_found_error: 179 exception = server_not_found_error 180 181 if exception: 182 if retry_num == num_retries: 183 raise exception 184 else: 185 continue 186 187 if not _should_retry_response(resp.status, content): 188 break 189 190 return resp, content

191

192 193 -class MediaUploadProgress(object):

194 """Status of a resumable upload.""" 195

196 - def __init__(self, resumable_progress, total_size):

197 """Constructor. 198 199 Args: 200 resumable_progress: int, bytes sent so far. 201 total_size: int, total bytes in complete upload, or None if the total 202 upload size isn't known ahead of time. 203 """ 204 self.resumable_progress = resumable_progress 205 self.total_size = total_size

206

207 - def progress(self):

208 """Percent of upload completed, as a float. 209 210 Returns: 211 the percentage complete as a float, returning 0.0 if the total size of 212 the upload is unknown. 213 """ 214 if self.total_size is not None and self.total_size != 0: 215 return float(self.resumable_progress) / float(self.total_size) 216 else: 217 return 0.0

218

219 220 -class MediaDownloadProgress(object):

221 """Status of a resumable download.""" 222

223 - def __init__(self, resumable_progress, total_size):

224 """Constructor. 225 226 Args: 227 resumable_progress: int, bytes received so far. 228 total_size: int, total bytes in complete download. 229 """ 230 self.resumable_progress = resumable_progress 231 self.total_size = total_size

232

233 - def progress(self):

234 """Percent of download completed, as a float. 235 236 Returns: 237 the percentage complete as a float, returning 0.0 if the total size of 238 the download is unknown. 239 """ 240 if self.total_size is not None and self.total_size != 0: 241 return float(self.resumable_progress) / float(self.total_size) 242 else: 243 return 0.0

244

245 246 -class MediaUpload(object):

247 """Describes a media object to upload. 248 249 Base class that defines the interface of MediaUpload subclasses. 250 251 Note that subclasses of MediaUpload may allow you to control the chunksize 252 when uploading a media object. It is important to keep the size of the chunk 253 as large as possible to keep the upload efficient. Other factors may influence 254 the size of the chunk you use, particularly if you are working in an 255 environment where individual HTTP requests may have a hardcoded time limit, 256 such as under certain classes of requests under Google App Engine. 257 258 Streams are io.Base compatible objects that support seek(). Some MediaUpload 259 subclasses support using streams directly to upload data. Support for 260 streaming may be indicated by a MediaUpload sub-class and if appropriate for a 261 platform that stream will be used for uploading the media object. The support 262 for streaming is indicated by has_stream() returning True. The stream() method 263 should return an io.Base object that supports seek(). On platforms where the 264 underlying httplib module supports streaming, for example Python 2.6 and 265 later, the stream will be passed into the http library which will result in 266 less memory being used and possibly faster uploads. 267 268 If you need to upload media that can't be uploaded using any of the existing 269 MediaUpload sub-class then you can sub-class MediaUpload for your particular 270 needs. 271 """ 272

273 - def chunksize(self):

274 """Chunk size for resumable uploads. 275 276 Returns: 277 Chunk size in bytes. 278 """ 279 raise NotImplementedError()

280

281 - def mimetype(self):

282 """Mime type of the body. 283 284 Returns: 285 Mime type. 286 """ 287 return 'application/octet-stream'

288

289 - def size(self):

290 """Size of upload. 291 292 Returns: 293 Size of the body, or None of the size is unknown. 294 """ 295 return None

296

297 - def resumable(self):

298 """Whether this upload is resumable. 299 300 Returns: 301 True if resumable upload or False. 302 """ 303 return False

304

305 - def getbytes(self, begin, end):

306 """Get bytes from the media. 307 308 Args: 309 begin: int, offset from beginning of file. 310 length: int, number of bytes to read, starting at begin. 311 312 Returns: 313 A string of bytes read. May be shorter than length if EOF was reached 314 first. 315 """ 316 raise NotImplementedError()

317

318 - def has_stream(self):

319 """Does the underlying upload support a streaming interface. 320 321 Streaming means it is an io.IOBase subclass that supports seek, i.e. 322 seekable() returns True. 323 324 Returns: 325 True if the call to stream() will return an instance of a seekable io.Base 326 subclass. 327 """ 328 return False

329

330 - def stream(self):

331 """A stream interface to the data being uploaded. 332 333 Returns: 334 The returned value is an io.IOBase subclass that supports seek, i.e. 335 seekable() returns True. 336 """ 337 raise NotImplementedError()

338 339 @util.positional(1)

340 - def _to_json(self, strip=None):

341 """Utility function for creating a JSON representation of a MediaUpload. 342 343 Args: 344 strip: array, An array of names of members to not include in the JSON. 345 346 Returns: 347 string, a JSON representation of this instance, suitable to pass to 348 from_json(). 349 """ 350 t = type(self) 351 d = copy.copy(self.__dict__) 352 if strip is not None: 353 for member in strip: 354 del d[member] 355 d['_class'] = t.__name__ 356 d['_module'] = t.__module__ 357 return json.dumps(d)

358

359 - def to_json(self):

360 """Create a JSON representation of an instance of MediaUpload. 361 362 Returns: 363 string, a JSON representation of this instance, suitable to pass to 364 from_json(). 365 """ 366 return self._to_json()

367 368 @classmethod

369 - def new_from_json(cls, s):

370 """Utility class method to instantiate a MediaUpload subclass from a JSON 371 representation produced by to_json(). 372 373 Args: 374 s: string, JSON from to_json(). 375 376 Returns: 377 An instance of the subclass of MediaUpload that was serialized with 378 to_json(). 379 """ 380 data = json.loads(s) 381 # Find and call the right classmethod from_json() to restore the object. 382 module = data['_module'] 383 m = __import__(module, fromlist=module.split('.')[:-1]) 384 kls = getattr(m, data['_class']) 385 from_json = getattr(kls, 'from_json') 386 return from_json(s)

387

388 389 -class MediaIoBaseUpload(MediaUpload):

390 """A MediaUpload for a io.Base objects. 391 392 Note that the Python file object is compatible with io.Base and can be used 393 with this class also. 394 395 fh = BytesIO('...Some data to upload...') 396 media = MediaIoBaseUpload(fh, mimetype='image/png', 397 chunksize=1024*1024, resumable=True) 398 farm.animals().insert( 399 id='cow', 400 name='cow.png', 401 media_body=media).execute() 402 403 Depending on the platform you are working on, you may pass -1 as the 404 chunksize, which indicates that the entire file should be uploaded in a single 405 request. If the underlying platform supports streams, such as Python 2.6 or 406 later, then this can be very efficient as it avoids multiple connections, and 407 also avoids loading the entire file into memory before sending it. Note that 408 Google App Engine has a 5MB limit on request size, so you should never set 409 your chunksize larger than 5MB, or to -1. 410 """ 411 412 @util.positional(3)

413 - def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, 414 resumable=False):

415 """Constructor. 416 417 Args: 418 fd: io.Base or file object, The source of the bytes to upload. MUST be 419 opened in blocking mode, do not use streams opened in non-blocking mode. 420 The given stream must be seekable, that is, it must be able to call 421 seek() on fd. 422 mimetype: string, Mime-type of the file. 423 chunksize: int, File will be uploaded in chunks of this many bytes. Only 424 used if resumable=True. Pass in a value of -1 if the file is to be 425 uploaded as a single chunk. Note that Google App Engine has a 5MB limit 426 on request size, so you should never set your chunksize larger than 5MB, 427 or to -1. 428 resumable: bool, True if this is a resumable upload. False means upload 429 in a single request. 430 """ 431 super(MediaIoBaseUpload, self).__init__() 432 self._fd = fd 433 self._mimetype = mimetype 434 if not (chunksize == -1 or chunksize > 0): 435 raise InvalidChunkSizeError() 436 self._chunksize = chunksize 437 self._resumable = resumable 438 439 self._fd.seek(0, os.SEEK_END) 440 self._size = self._fd.tell()

441

442 - def chunksize(self):

443 """Chunk size for resumable uploads. 444 445 Returns: 446 Chunk size in bytes. 447 """ 448 return self._chunksize

449

450 - def mimetype(self):

451 """Mime type of the body. 452 453 Returns: 454 Mime type. 455 """ 456 return self._mimetype

457

458 - def size(self):

459 """Size of upload. 460 461 Returns: 462 Size of the body, or None of the size is unknown. 463 """ 464 return self._size

465

466 - def resumable(self):

467 """Whether this upload is resumable. 468 469 Returns: 470 True if resumable upload or False. 471 """ 472 return self._resumable

473

474 - def getbytes(self, begin, length):

475 """Get bytes from the media. 476 477 Args: 478 begin: int, offset from beginning of file. 479 length: int, number of bytes to read, starting at begin. 480 481 Returns: 482 A string of bytes read. May be shorted than length if EOF was reached 483 first. 484 """ 485 self._fd.seek(begin) 486 return self._fd.read(length)

487

488 - def has_stream(self):

489 """Does the underlying upload support a streaming interface. 490 491 Streaming means it is an io.IOBase subclass that supports seek, i.e. 492 seekable() returns True. 493 494 Returns: 495 True if the call to stream() will return an instance of a seekable io.Base 496 subclass. 497 """ 498 return True

499

500 - def stream(self):

501 """A stream interface to the data being uploaded. 502 503 Returns: 504 The returned value is an io.IOBase subclass that supports seek, i.e. 505 seekable() returns True. 506 """ 507 return self._fd

508

509 - def to_json(self):

510 """This upload type is not serializable.""" 511 raise NotImplementedError('MediaIoBaseUpload is not serializable.')

512

513 514 -class MediaFileUpload(MediaIoBaseUpload):

515 """A MediaUpload for a file. 516 517 Construct a MediaFileUpload and pass as the media_body parameter of the 518 method. For example, if we had a service that allowed uploading images: 519 520 media = MediaFileUpload('cow.png', mimetype='image/png', 521 chunksize=1024*1024, resumable=True) 522 farm.animals().insert( 523 id='cow', 524 name='cow.png', 525 media_body=media).execute() 526 527 Depending on the platform you are working on, you may pass -1 as the 528 chunksize, which indicates that the entire file should be uploaded in a single 529 request. If the underlying platform supports streams, such as Python 2.6 or 530 later, then this can be very efficient as it avoids multiple connections, and 531 also avoids loading the entire file into memory before sending it. Note that 532 Google App Engine has a 5MB limit on request size, so you should never set 533 your chunksize larger than 5MB, or to -1. 534 """ 535 536 @util.positional(2)

537 - def __init__(self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, 538 resumable=False):

539 """Constructor. 540 541 Args: 542 filename: string, Name of the file. 543 mimetype: string, Mime-type of the file. If None then a mime-type will be 544 guessed from the file extension. 545 chunksize: int, File will be uploaded in chunks of this many bytes. Only 546 used if resumable=True. Pass in a value of -1 if the file is to be 547 uploaded in a single chunk. Note that Google App Engine has a 5MB limit 548 on request size, so you should never set your chunksize larger than 5MB, 549 or to -1. 550 resumable: bool, True if this is a resumable upload. False means upload 551 in a single request. 552 """ 553 self._filename = filename 554 fd = open(self._filename, 'rb') 555 if mimetype is None: 556 # No mimetype provided, make a guess. 557 mimetype, _ = mimetypes.guess_type(filename) 558 if mimetype is None: 559 # Guess failed, use octet-stream. 560 mimetype = 'application/octet-stream' 561 super(MediaFileUpload, self).__init__(fd, mimetype, chunksize=chunksize, 562 resumable=resumable)

563

564 - def to_json(self):

565 """Creating a JSON representation of an instance of MediaFileUpload. 566 567 Returns: 568 string, a JSON representation of this instance, suitable to pass to 569 from_json(). 570 """ 571 return self._to_json(strip=['_fd'])

572 573 @staticmethod

574 - def from_json(s):

575 d = json.loads(s) 576 return MediaFileUpload(d['_filename'], mimetype=d['_mimetype'], 577 chunksize=d['_chunksize'], resumable=d['_resumable'])

578

579 580 -class MediaInMemoryUpload(MediaIoBaseUpload):

581 """MediaUpload for a chunk of bytes. 582 583 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 584 the stream. 585 """ 586 587 @util.positional(2)

588 - def __init__(self, body, mimetype='application/octet-stream', 589 chunksize=DEFAULT_CHUNK_SIZE, resumable=False):

590 """Create a new MediaInMemoryUpload. 591 592 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 593 the stream. 594 595 Args: 596 body: string, Bytes of body content. 597 mimetype: string, Mime-type of the file or default of 598 'application/octet-stream'. 599 chunksize: int, File will be uploaded in chunks of this many bytes. Only 600 used if resumable=True. 601 resumable: bool, True if this is a resumable upload. False means upload 602 in a single request. 603 """ 604 fd = BytesIO(body) 605 super(MediaInMemoryUpload, self).__init__(fd, mimetype, chunksize=chunksize, 606 resumable=resumable)

607

608 609 -class MediaIoBaseDownload(object):

610 """"Download media resources. 611 612 Note that the Python file object is compatible with io.Base and can be used 613 with this class also. 614 615 616 Example: 617 request = farms.animals().get_media(id='cow') 618 fh = io.FileIO('cow.png', mode='wb') 619 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 620 621 done = False 622 while done is False: 623 status, done = downloader.next_chunk() 624 if status: 625 print "Download %d%%." % int(status.progress() * 100) 626 print "Download Complete!" 627 """ 628 629 @util.positional(3)

630 - def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE):

631 """Constructor. 632 633 Args: 634 fd: io.Base or file object, The stream in which to write the downloaded 635 bytes. 636 request: googleapiclient.http.HttpRequest, the media request to perform in 637 chunks. 638 chunksize: int, File will be downloaded in chunks of this many bytes. 639 """ 640 self._fd = fd 641 self._request = request 642 self._uri = request.uri 643 self._chunksize = chunksize 644 self._progress = 0 645 self._total_size = None 646 self._done = False 647 648 # Stubs for testing. 649 self._sleep = time.sleep 650 self._rand = random.random 651 652 self._headers = {} 653 for k, v in six.iteritems(request.headers): 654 # allow users to supply custom headers by setting them on the request 655 # but strip out the ones that are set by default on requests generated by 656 # API methods like Drive's files().get(fileId=...) 657 if not k.lower() in ('accept', 'accept-encoding', 'user-agent'): 658 self._headers[k] = v

659 660 @util.positional(1)

661 - def next_chunk(self, num_retries=0):

662 """Get the next chunk of the download. 663 664 Args: 665 num_retries: Integer, number of times to retry with randomized 666 exponential backoff. If all retries fail, the raised HttpError 667 represents the last request. If zero (default), we attempt the 668 request only once. 669 670 Returns: 671 (status, done): (MediaDownloadProgress, boolean) 672 The value of 'done' will be True when the media has been fully 673 downloaded or the total size of the media is unknown. 674 675 Raises: 676 googleapiclient.errors.HttpError if the response was not a 2xx. 677 httplib2.HttpLib2Error if a transport error has occured. 678 """ 679 headers = self._headers.copy() 680 headers['range'] = 'bytes=%d-%d' % ( 681 self._progress, self._progress + self._chunksize) 682 http = self._request.http 683 684 resp, content = _retry_request( 685 http, num_retries, 'media download', self._sleep, self._rand, self._uri, 686 'GET', headers=headers) 687 688 if resp.status in [200, 206]: 689 if 'content-location' in resp and resp['content-location'] != self._uri: 690 self._uri = resp['content-location'] 691 self._progress += len(content) 692 self._fd.write(content) 693 694 if 'content-range' in resp: 695 content_range = resp['content-range'] 696 length = content_range.rsplit('/', 1)[1] 697 self._total_size = int(length) 698 elif 'content-length' in resp: 699 self._total_size = int(resp['content-length']) 700 701 if self._total_size is None or self._progress == self._total_size: 702 self._done = True 703 return MediaDownloadProgress(self._progress, self._total_size), self._done 704 else: 705 raise HttpError(resp, content, uri=self._uri)

706

707 708 -class _StreamSlice(object):

709 """Truncated stream. 710 711 Takes a stream and presents a stream that is a slice of the original stream. 712 This is used when uploading media in chunks. In later versions of Python a 713 stream can be passed to httplib in place of the string of data to send. The 714 problem is that httplib just blindly reads to the end of the stream. This 715 wrapper presents a virtual stream that only reads to the end of the chunk. 716 """ 717

718 - def __init__(self, stream, begin, chunksize):

719 """Constructor. 720 721 Args: 722 stream: (io.Base, file object), the stream to wrap. 723 begin: int, the seek position the chunk begins at. 724 chunksize: int, the size of the chunk. 725 """ 726 self._stream = stream 727 self._begin = begin 728 self._chunksize = chunksize 729 self._stream.seek(begin)

730

731 - def read(self, n=-1):

732 """Read n bytes. 733 734 Args: 735 n, int, the number of bytes to read. 736 737 Returns: 738 A string of length 'n', or less if EOF is reached. 739 """ 740 # The data left available to read sits in [cur, end) 741 cur = self._stream.tell() 742 end = self._begin + self._chunksize 743 if n == -1 or cur + n > end: 744 n = end - cur 745 return self._stream.read(n)

746

747 748 -class HttpRequest(object):

749 """Encapsulates a single HTTP request.""" 750 751 @util.positional(4)

752 - def __init__(self, http, postproc, uri, 753 method='GET', 754 body=None, 755 headers=None, 756 methodId=None, 757 resumable=None):

758 """Constructor for an HttpRequest. 759 760 Args: 761 http: httplib2.Http, the transport object to use to make a request 762 postproc: callable, called on the HTTP response and content to transform 763 it into a data object before returning, or raising an exception 764 on an error. 765 uri: string, the absolute URI to send the request to 766 method: string, the HTTP method to use 767 body: string, the request body of the HTTP request, 768 headers: dict, the HTTP request headers 769 methodId: string, a unique identifier for the API method being called. 770 resumable: MediaUpload, None if this is not a resumbale request. 771 """ 772 self.uri = uri 773 self.method = method 774 self.body = body 775 self.headers = headers or {} 776 self.methodId = methodId 777 self.http = http 778 self.postproc = postproc 779 self.resumable = resumable 780 self.response_callbacks = [] 781 self._in_error_state = False 782 783 # The size of the non-media part of the request. 784 self.body_size = len(self.body or '') 785 786 # The resumable URI to send chunks to. 787 self.resumable_uri = None 788 789 # The bytes that have been uploaded. 790 self.resumable_progress = 0 791 792 # Stubs for testing. 793 self._rand = random.random 794 self._sleep = time.sleep

795 796 @util.positional(1)

797 - def execute(self, http=None, num_retries=0):

798 """Execute the request. 799 800 Args: 801 http: httplib2.Http, an http object to be used in place of the 802 one the HttpRequest request object was constructed with. 803 num_retries: Integer, number of times to retry with randomized 804 exponential backoff. If all retries fail, the raised HttpError 805 represents the last request. If zero (default), we attempt the 806 request only once. 807 808 Returns: 809 A deserialized object model of the response body as determined 810 by the postproc. 811 812 Raises: 813 googleapiclient.errors.HttpError if the response was not a 2xx. 814 httplib2.HttpLib2Error if a transport error has occured. 815 """ 816 if http is None: 817 http = self.http 818 819 if self.resumable: 820 body = None 821 while body is None: 822 _, body = self.next_chunk(http=http, num_retries=num_retries) 823 return body 824 825 # Non-resumable case. 826 827 if 'content-length' not in self.headers: 828 self.headers['content-length'] = str(self.body_size) 829 # If the request URI is too long then turn it into a POST request. 830 # Assume that a GET request never contains a request body. 831 if len(self.uri) > MAX_URI_LENGTH and self.method == 'GET': 832 self.method = 'POST' 833 self.headers['x-http-method-override'] = 'GET' 834 self.headers['content-type'] = 'application/x-www-form-urlencoded' 835 parsed = urlparse(self.uri) 836 self.uri = urlunparse( 837 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, 838 None) 839 ) 840 self.body = parsed.query 841 self.headers['content-length'] = str(len(self.body)) 842 843 # Handle retries for server-side errors. 844 resp, content = _retry_request( 845 http, num_retries, 'request', self._sleep, self._rand, str(self.uri), 846 method=str(self.method), body=self.body, headers=self.headers) 847 848 for callback in self.response_callbacks: 849 callback(resp) 850 if resp.status >= 300: 851 raise HttpError(resp, content, uri=self.uri) 852 return self.postproc(resp, content)

853 854 @util.positional(2)

855 - def add_response_callback(self, cb):

856 """add_response_headers_callback 857 858 Args: 859 cb: Callback to be called on receiving the response headers, of signature: 860 861 def cb(resp): 862 # Where resp is an instance of httplib2.Response 863 """ 864 self.response_callbacks.append(cb)

865 866 @util.positional(1)

867 - def next_chunk(self, http=None, num_retries=0):

868 """Execute the next step of a resumable upload. 869 870 Can only be used if the method being executed supports media uploads and 871 the MediaUpload object passed in was flagged as using resumable upload. 872 873 Example: 874 875 media = MediaFileUpload('cow.png', mimetype='image/png', 876 chunksize=1000, resumable=True) 877 request = farm.animals().insert( 878 id='cow', 879 name='cow.png', 880 media_body=media) 881 882 response = None 883 while response is None: 884 status, response = request.next_chunk() 885 if status: 886 print "Upload %d%% complete." % int(status.progress() * 100) 887 888 889 Args: 890 http: httplib2.Http, an http object to be used in place of the 891 one the HttpRequest request object was constructed with. 892 num_retries: Integer, number of times to retry with randomized 893 exponential backoff. If all retries fail, the raised HttpError 894 represents the last request. If zero (default), we attempt the 895 request only once. 896 897 Returns: 898 (status, body): (ResumableMediaStatus, object) 899 The body will be None until the resumable media is fully uploaded. 900 901 Raises: 902 googleapiclient.errors.HttpError if the response was not a 2xx. 903 httplib2.HttpLib2Error if a transport error has occured. 904 """ 905 if http is None: 906 http = self.http 907 908 if self.resumable.size() is None: 909 size = '*' 910 else: 911 size = str(self.resumable.size()) 912 913 if self.resumable_uri is None: 914 start_headers = copy.copy(self.headers) 915 start_headers['X-Upload-Content-Type'] = self.resumable.mimetype() 916 if size != '*': 917 start_headers['X-Upload-Content-Length'] = size 918 start_headers['content-length'] = str(self.body_size) 919 920 resp, content = _retry_request( 921 http, num_retries, 'resumable URI request', self._sleep, self._rand, 922 self.uri, method=self.method, body=self.body, headers=start_headers) 923 924 if resp.status == 200 and 'location' in resp: 925 self.resumable_uri = resp['location'] 926 else: 927 raise ResumableUploadError(resp, content) 928 elif self._in_error_state: 929 # If we are in an error state then query the server for current state of 930 # the upload by sending an empty PUT and reading the 'range' header in 931 # the response. 932 headers = { 933 'Content-Range': 'bytes */%s' % size, 934 'content-length': '0' 935 } 936 resp, content = http.request(self.resumable_uri, 'PUT', 937 headers=headers) 938 status, body = self._process_response(resp, content) 939 if body: 940 # The upload was complete. 941 return (status, body) 942 943 if self.resumable.has_stream(): 944 data = self.resumable.stream() 945 if self.resumable.chunksize() == -1: 946 data.seek(self.resumable_progress) 947 chunk_end = self.resumable.size() - self.resumable_progress - 1 948 else: 949 # Doing chunking with a stream, so wrap a slice of the stream. 950 data = _StreamSlice(data, self.resumable_progress, 951 self.resumable.chunksize()) 952 chunk_end = min( 953 self.resumable_progress + self.resumable.chunksize() - 1, 954 self.resumable.size() - 1) 955 else: 956 data = self.resumable.getbytes( 957 self.resumable_progress, self.resumable.chunksize()) 958 959 # A short read implies that we are at EOF, so finish the upload. 960 if len(data) < self.resumable.chunksize(): 961 size = str(self.resumable_progress + len(data)) 962 963 chunk_end = self.resumable_progress + len(data) - 1 964 965 headers = { 966 'Content-Range': 'bytes %d-%d/%s' % ( 967 self.resumable_progress, chunk_end, size), 968 # Must set the content-length header here because httplib can't 969 # calculate the size when working with _StreamSlice. 970 'Content-Length': str(chunk_end - self.resumable_progress + 1) 971 } 972 973 for retry_num in range(num_retries + 1): 974 if retry_num > 0: 975 self._sleep(self._rand() * 2**retry_num) 976 LOGGER.warning( 977 'Retry #%d for media upload: %s %s, following status: %d' 978 % (retry_num, self.method, self.uri, resp.status)) 979 980 try: 981 resp, content = http.request(self.resumable_uri, method='PUT', 982 body=data, 983 headers=headers) 984 except: 985 self._in_error_state = True 986 raise 987 if not _should_retry_response(resp.status, content): 988 break 989 990 return self._process_response(resp, content)

991

992 - def _process_response(self, resp, content):

993 """Process the response from a single chunk upload. 994 995 Args: 996 resp: httplib2.Response, the response object. 997 content: string, the content of the response. 998 999 Returns: 1000 (status, body): (ResumableMediaStatus, object) 1001 The body will be None until the resumable media is fully uploaded. 1002 1003 Raises: 1004 googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 1005 """ 1006 if resp.status in [200, 201]: 1007 self._in_error_state = False 1008 return None, self.postproc(resp, content) 1009 elif resp.status == 308: 1010 self._in_error_state = False 1011 # A "308 Resume Incomplete" indicates we are not done. 1012 try: 1013 self.resumable_progress = int(resp['range'].split('-')[1]) + 1 1014 except KeyError: 1015 # If resp doesn't contain range header, resumable progress is 0 1016 self.resumable_progress = 0 1017 if 'location' in resp: 1018 self.resumable_uri = resp['location'] 1019 else: 1020 self._in_error_state = True 1021 raise HttpError(resp, content, uri=self.uri) 1022 1023 return (MediaUploadProgress(self.resumable_progress, self.resumable.size()), 1024 None)

1025

1026 - def to_json(self):

1027 """Returns a JSON representation of the HttpRequest.""" 1028 d = copy.copy(self.__dict__) 1029 if d['resumable'] is not None: 1030 d['resumable'] = self.resumable.to_json() 1031 del d['http'] 1032 del d['postproc'] 1033 del d['_sleep'] 1034 del d['_rand'] 1035 1036 return json.dumps(d)

1037 1038 @staticmethod

1039 - def from_json(s, http, postproc):

1040 """Returns an HttpRequest populated with info from a JSON object.""" 1041 d = json.loads(s) 1042 if d['resumable'] is not None: 1043 d['resumable'] = MediaUpload.new_from_json(d['resumable']) 1044 return HttpRequest( 1045 http, 1046 postproc, 1047 uri=d['uri'], 1048 method=d['method'], 1049 body=d['body'], 1050 headers=d['headers'], 1051 methodId=d['methodId'], 1052 resumable=d['resumable'])

1053

1054 1055 -class BatchHttpRequest(object):

1056 """Batches multiple HttpRequest objects into a single HTTP request. 1057 1058 Example: 1059 from googleapiclient.http import BatchHttpRequest 1060 1061 def list_animals(request_id, response, exception): 1062 \"\"\"Do something with the animals list response.\"\"\" 1063 if exception is not None: 1064 # Do something with the exception. 1065 pass 1066 else: 1067 # Do something with the response. 1068 pass 1069 1070 def list_farmers(request_id, response, exception): 1071 \"\"\"Do something with the farmers list response.\"\"\" 1072 if exception is not None: 1073 # Do something with the exception. 1074 pass 1075 else: 1076 # Do something with the response. 1077 pass 1078 1079 service = build('farm', 'v2') 1080 1081 batch = BatchHttpRequest() 1082 1083 batch.add(service.animals().list(), list_animals) 1084 batch.add(service.farmers().list(), list_farmers) 1085 batch.execute(http=http) 1086 """ 1087 1088 @util.positional(1)

1089 - def __init__(self, callback=None, batch_uri=None):

1090 """Constructor for a BatchHttpRequest. 1091 1092 Args: 1093 callback: callable, A callback to be called for each response, of the 1094 form callback(id, response, exception). The first parameter is the 1095 request id, and the second is the deserialized response object. The 1096 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1097 occurred while processing the request, or None if no error occurred. 1098 batch_uri: string, URI to send batch requests to. 1099 """ 1100 if batch_uri is None: 1101 batch_uri = _LEGACY_BATCH_URI 1102 1103 if batch_uri == _LEGACY_BATCH_URI: 1104 LOGGER.warn( 1105 "You have constructed a BatchHttpRequest using the legacy batch " 1106 "endpoint %s. This endpoint will be turned down on March 25, 2019. " 1107 "Please provide the API-specific endpoint or use " 1108 "service.new_batch_http_request(). For more details see " 1109 "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" 1110 "and https://developers.google.com/api-client-library/python/guide/batch.", 1111 _LEGACY_BATCH_URI) 1112 self._batch_uri = batch_uri 1113 1114 # Global callback to be called for each individual response in the batch. 1115 self._callback = callback 1116 1117 # A map from id to request. 1118 self._requests = {} 1119 1120 # A map from id to callback. 1121 self._callbacks = {} 1122 1123 # List of request ids, in the order in which they were added. 1124 self._order = [] 1125 1126 # The last auto generated id. 1127 self._last_auto_id = 0 1128 1129 # Unique ID on which to base the Content-ID headers. 1130 self._base_id = None 1131 1132 # A map from request id to (httplib2.Response, content) response pairs 1133 self._responses = {} 1134 1135 # A map of id(Credentials) that have been refreshed. 1136 self._refreshed_credentials = {}

1137

1138 - def _refresh_and_apply_credentials(self, request, http):

1139 """Refresh the credentials and apply to the request. 1140 1141 Args: 1142 request: HttpRequest, the request. 1143 http: httplib2.Http, the global http object for the batch. 1144 """ 1145 # For the credentials to refresh, but only once per refresh_token 1146 # If there is no http per the request then refresh the http passed in 1147 # via execute() 1148 creds = None 1149 request_credentials = False 1150 1151 if request.http is not None: 1152 creds = _auth.get_credentials_from_http(request.http) 1153 request_credentials = True 1154 1155 if creds is None and http is not None: 1156 creds = _auth.get_credentials_from_http(http) 1157 1158 if creds is not None: 1159 if id(creds) not in self._refreshed_credentials: 1160 _auth.refresh_credentials(creds) 1161 self._refreshed_credentials[id(creds)] = 1 1162 1163 # Only apply the credentials if we are using the http object passed in, 1164 # otherwise apply() will get called during _serialize_request(). 1165 if request.http is None or not request_credentials: 1166 _auth.apply_credentials(creds, request.headers)

1167 1168

1169 - def _id_to_header(self, id_):

1170 """Convert an id to a Content-ID header value. 1171 1172 Args: 1173 id_: string, identifier of individual request. 1174 1175 Returns: 1176 A Content-ID header with the id_ encoded into it. A UUID is prepended to 1177 the value because Content-ID headers are supposed to be universally 1178 unique. 1179 """ 1180 if self._base_id is None: 1181 self._base_id = uuid.uuid4() 1182 1183 # NB: we intentionally leave whitespace between base/id and '+', so RFC2822 1184 # line folding works properly on Python 3; see 1185 # https://github.com/google/google-api-python-client/issues/164 1186 return '<%s + %s>' % (self._base_id, quote(id_))

1187

1188 - def _header_to_id(self, header):

1189 """Convert a Content-ID header value to an id. 1190 1191 Presumes the Content-ID header conforms to the format that _id_to_header() 1192 returns. 1193 1194 Args: 1195 header: string, Content-ID header value. 1196 1197 Returns: 1198 The extracted id value. 1199 1200 Raises: 1201 BatchError if the header is not in the expected format. 1202 """ 1203 if header[0] != '<' or header[-1] != '>': 1204 raise BatchError("Invalid value for Content-ID: %s" % header) 1205 if '+' not in header: 1206 raise BatchError("Invalid value for Content-ID: %s" % header) 1207 base, id_ = header[1:-1].split(' + ', 1) 1208 1209 return unquote(id_)

1210

1211 - def _serialize_request(self, request):

1212 """Convert an HttpRequest object into a string. 1213 1214 Args: 1215 request: HttpRequest, the request to serialize. 1216 1217 Returns: 1218 The request as a string in application/http format. 1219 """ 1220 # Construct status line 1221 parsed = urlparse(request.uri) 1222 request_line = urlunparse( 1223 ('', '', parsed.path, parsed.params, parsed.query, '') 1224 ) 1225 status_line = request.method + ' ' + request_line + ' HTTP/1.1\n' 1226 major, minor = request.headers.get('content-type', 'application/json').split('/') 1227 msg = MIMENonMultipart(major, minor) 1228 headers = request.headers.copy() 1229 1230 if request.http is not None: 1231 credentials = _auth.get_credentials_from_http(request.http) 1232 if credentials is not None: 1233 _auth.apply_credentials(credentials, headers) 1234 1235 # MIMENonMultipart adds its own Content-Type header. 1236 if 'content-type' in headers: 1237 del headers['content-type'] 1238 1239 for key, value in six.iteritems(headers): 1240 msg[key] = value 1241 msg['Host'] = parsed.netloc 1242 msg.set_unixfrom(None) 1243 1244 if request.body is not None: 1245 msg.set_payload(request.body) 1246 msg['content-length'] = str(len(request.body)) 1247 1248 # Serialize the mime message. 1249 fp = StringIO() 1250 # maxheaderlen=0 means don't line wrap headers. 1251 g = Generator(fp, maxheaderlen=0) 1252 g.flatten(msg, unixfrom=False) 1253 body = fp.getvalue() 1254 1255 return status_line + body

1256

1257 - def _deserialize_response(self, payload):

1258 """Convert string into httplib2 response and content. 1259 1260 Args: 1261 payload: string, headers and body as a string. 1262 1263 Returns: 1264 A pair (resp, content), such as would be returned from httplib2.request. 1265 """ 1266 # Strip off the status line 1267 status_line, payload = payload.split('\n', 1) 1268 protocol, status, reason = status_line.split(' ', 2) 1269 1270 # Parse the rest of the response 1271 parser = FeedParser() 1272 parser.feed(payload) 1273 msg = parser.close() 1274 msg['status'] = status 1275 1276 # Create httplib2.Response from the parsed headers. 1277 resp = httplib2.Response(msg) 1278 resp.reason = reason 1279 resp.version = int(protocol.split('/', 1)[1].replace('.', '')) 1280 1281 content = payload.split('\r\n\r\n', 1)[1] 1282 1283 return resp, content

1284

1285 - def _new_id(self):

1286 """Create a new id. 1287 1288 Auto incrementing number that avoids conflicts with ids already used. 1289 1290 Returns: 1291 string, a new unique id. 1292 """ 1293 self._last_auto_id += 1 1294 while str(self._last_auto_id) in self._requests: 1295 self._last_auto_id += 1 1296 return str(self._last_auto_id)

1297 1298 @util.positional(2)

1299 - def add(self, request, callback=None, request_id=None):

1300 """Add a new request. 1301 1302 Every callback added will be paired with a unique id, the request_id. That 1303 unique id will be passed back to the callback when the response comes back 1304 from the server. The default behavior is to have the library generate it's 1305 own unique id. If the caller passes in a request_id then they must ensure 1306 uniqueness for each request_id, and if they are not an exception is 1307 raised. Callers should either supply all request_ids or never supply a 1308 request id, to avoid such an error. 1309 1310 Args: 1311 request: HttpRequest, Request to add to the batch. 1312 callback: callable, A callback to be called for this response, of the 1313 form callback(id, response, exception). The first parameter is the 1314 request id, and the second is the deserialized response object. The 1315 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1316 occurred while processing the request, or None if no errors occurred. 1317 request_id: string, A unique id for the request. The id will be passed 1318 to the callback with the response. 1319 1320 Returns: 1321 None 1322 1323 Raises: 1324 BatchError if a media request is added to a batch. 1325 KeyError is the request_id is not unique. 1326 """ 1327 1328 if len(self._order) >= MAX_BATCH_LIMIT: 1329 raise BatchError("Exceeded the maximum calls(%d) in a single batch request." 1330 % MAX_BATCH_LIMIT) 1331 if request_id is None: 1332 request_id = self._new_id() 1333 if request.resumable is not None: 1334 raise BatchError("Media requests cannot be used in a batch request.") 1335 if request_id in self._requests: 1336 raise KeyError("A request with this ID already exists: %s" % request_id) 1337 self._requests[request_id] = request 1338 self._callbacks[request_id] = callback 1339 self._order.append(request_id)

1340

1341 - def _execute(self, http, order, requests):

1342 """Serialize batch request, send to server, process response. 1343 1344 Args: 1345 http: httplib2.Http, an http object to be used to make the request with. 1346 order: list, list of request ids in the order they were added to the 1347 batch. 1348 request: list, list of request objects to send. 1349 1350 Raises: 1351 httplib2.HttpLib2Error if a transport error has occured. 1352 googleapiclient.errors.BatchError if the response is the wrong format. 1353 """ 1354 message = MIMEMultipart('mixed') 1355 # Message should not write out it's own headers. 1356 setattr(message, '_write_headers', lambda self: None) 1357 1358 # Add all the individual requests. 1359 for request_id in order: 1360 request = requests[request_id] 1361 1362 msg = MIMENonMultipart('application', 'http') 1363 msg['Content-Transfer-Encoding'] = 'binary' 1364 msg['Content-ID'] = self._id_to_header(request_id) 1365 1366 body = self._serialize_request(request) 1367 msg.set_payload(body) 1368 message.attach(msg) 1369 1370 # encode the body: note that we can't use `as_string`, because 1371 # it plays games with `From ` lines. 1372 fp = StringIO() 1373 g = Generator(fp, mangle_from_=False) 1374 g.flatten(message, unixfrom=False) 1375 body = fp.getvalue() 1376 1377 headers = {} 1378 headers['content-type'] = ('multipart/mixed; ' 1379 'boundary="%s"') % message.get_boundary() 1380 1381 resp, content = http.request(self._batch_uri, method='POST', body=body, 1382 headers=headers) 1383 1384 if resp.status >= 300: 1385 raise HttpError(resp, content, uri=self._batch_uri) 1386 1387 # Prepend with a content-type header so FeedParser can handle it. 1388 header = 'content-type: %s\r\n\r\n' % resp['content-type'] 1389 # PY3's FeedParser only accepts unicode. So we should decode content 1390 # here, and encode each payload again. 1391 if six.PY3: 1392 content = content.decode('utf-8') 1393 for_parser = header + content 1394 1395 parser = FeedParser() 1396 parser.feed(for_parser) 1397 mime_response = parser.close() 1398 1399 if not mime_response.is_multipart(): 1400 raise BatchError("Response not in multipart/mixed format.", resp=resp, 1401 content=content) 1402 1403 for part in mime_response.get_payload(): 1404 request_id = self._header_to_id(part['Content-ID']) 1405 response, content = self._deserialize_response(part.get_payload()) 1406 # We encode content here to emulate normal http response. 1407 if isinstance(content, six.text_type): 1408 content = content.encode('utf-8') 1409 self._responses[request_id] = (response, content)

1410 1411 @util.positional(1)

1412 - def execute(self, http=None):

1413 """Execute all the requests as a single batched HTTP request. 1414 1415 Args: 1416 http: httplib2.Http, an http object to be used in place of the one the 1417 HttpRequest request object was constructed with. If one isn't supplied 1418 then use a http object from the requests in this batch. 1419 1420 Returns: 1421 None 1422 1423 Raises: 1424 httplib2.HttpLib2Error if a transport error has occured. 1425 googleapiclient.errors.BatchError if the response is the wrong format. 1426 """ 1427 # If we have no requests return 1428 if len(self._order) == 0: 1429 return None 1430 1431 # If http is not supplied use the first valid one given in the requests. 1432 if http is None: 1433 for request_id in self._order: 1434 request = self._requests[request_id] 1435 if request is not None: 1436 http = request.http 1437 break 1438 1439 if http is None: 1440 raise ValueError("Missing a valid http object.") 1441 1442 # Special case for OAuth2Credentials-style objects which have not yet been 1443 # refreshed with an initial access_token. 1444 creds = _auth.get_credentials_from_http(http) 1445 if creds is not None: 1446 if not _auth.is_valid(creds): 1447 LOGGER.info('Attempting refresh to obtain initial access_token') 1448 _auth.refresh_credentials(creds) 1449 1450 self._execute(http, self._order, self._requests) 1451 1452 # Loop over all the requests and check for 401s. For each 401 request the 1453 # credentials should be refreshed and then sent again in a separate batch. 1454 redo_requests = {} 1455 redo_order = [] 1456 1457 for request_id in self._order: 1458 resp, content = self._responses[request_id] 1459 if resp['status'] == '401': 1460 redo_order.append(request_id) 1461 request = self._requests[request_id] 1462 self._refresh_and_apply_credentials(request, http) 1463 redo_requests[request_id] = request 1464 1465 if redo_requests: 1466 self._execute(http, redo_order, redo_requests) 1467 1468 # Now process all callbacks that are erroring, and raise an exception for 1469 # ones that return a non-2xx response? Or add extra parameter to callback 1470 # that contains an HttpError? 1471 1472 for request_id in self._order: 1473 resp, content = self._responses[request_id] 1474 1475 request = self._requests[request_id] 1476 callback = self._callbacks[request_id] 1477 1478 response = None 1479 exception = None 1480 try: 1481 if resp.status >= 300: 1482 raise HttpError(resp, content, uri=request.uri) 1483 response = request.postproc(resp, content) 1484 except HttpError as e: 1485 exception = e 1486 1487 if callback is not None: 1488 callback(request_id, response, exception) 1489 if self._callback is not None: 1490 self._callback(request_id, response, exception)

1491

1492 1493 -class HttpRequestMock(object):

1494 """Mock of HttpRequest. 1495 1496 Do not construct directly, instead use RequestMockBuilder. 1497 """ 1498

1499 - def __init__(self, resp, content, postproc):

1500 """Constructor for HttpRequestMock 1501 1502 Args: 1503 resp: httplib2.Response, the response to emulate coming from the request 1504 content: string, the response body 1505 postproc: callable, the post processing function usually supplied by 1506 the model class. See model.JsonModel.response() as an example. 1507 """ 1508 self.resp = resp 1509 self.content = content 1510 self.postproc = postproc 1511 if resp is None: 1512 self.resp = httplib2.Response({'status': 200, 'reason': 'OK'}) 1513 if 'reason' in self.resp: 1514 self.resp.reason = self.resp['reason']

1515

1516 - def execute(self, http=None):

1517 """Execute the request. 1518 1519 Same behavior as HttpRequest.execute(), but the response is 1520 mocked and not really from an HTTP request/response. 1521 """ 1522 return self.postproc(self.resp, self.content)

1523

1524 1525 -class RequestMockBuilder(object):

1526 """A simple mock of HttpRequest 1527 1528 Pass in a dictionary to the constructor that maps request methodIds to 1529 tuples of (httplib2.Response, content, opt_expected_body) that should be 1530 returned when that method is called. None may also be passed in for the 1531 httplib2.Response, in which case a 200 OK response will be generated. 1532 If an opt_expected_body (str or dict) is provided, it will be compared to 1533 the body and UnexpectedBodyError will be raised on inequality. 1534 1535 Example: 1536 response = '{"data": {"id": "tag:google.c...' 1537 requestBuilder = RequestMockBuilder( 1538 { 1539 'plus.activities.get': (None, response), 1540 } 1541 ) 1542 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1543 1544 Methods that you do not supply a response for will return a 1545 200 OK with an empty string as the response content or raise an excpetion 1546 if check_unexpected is set to True. The methodId is taken from the rpcName 1547 in the discovery document. 1548 1549 For more details see the project wiki. 1550 """ 1551

1552 - def __init__(self, responses, check_unexpected=False):

1553 """Constructor for RequestMockBuilder 1554 1555 The constructed object should be a callable object 1556 that can replace the class HttpResponse. 1557 1558 responses - A dictionary that maps methodIds into tuples 1559 of (httplib2.Response, content). The methodId 1560 comes from the 'rpcName' field in the discovery 1561 document. 1562 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1563 should be raised on unsupplied method. 1564 """ 1565 self.responses = responses 1566 self.check_unexpected = check_unexpected

1567

1568 - def __call__(self, http, postproc, uri, method='GET', body=None, 1569 headers=None, methodId=None, resumable=None):

1570 """Implements the callable interface that discovery.build() expects 1571 of requestBuilder, which is to build an object compatible with 1572 HttpRequest.execute(). See that method for the description of the 1573 parameters and the expected response. 1574 """ 1575 if methodId in self.responses: 1576 response = self.responses[methodId] 1577 resp, content = response[:2] 1578 if len(response) > 2: 1579 # Test the body against the supplied expected_body. 1580 expected_body = response[2] 1581 if bool(expected_body) != bool(body): 1582 # Not expecting a body and provided one 1583 # or expecting a body and not provided one. 1584 raise UnexpectedBodyError(expected_body, body) 1585 if isinstance(expected_body, str): 1586 expected_body = json.loads(expected_body) 1587 body = json.loads(body) 1588 if body != expected_body: 1589 raise UnexpectedBodyError(expected_body, body) 1590 return HttpRequestMock(resp, content, postproc) 1591 elif self.check_unexpected: 1592 raise UnexpectedMethodError(methodId=methodId) 1593 else: 1594 model = JsonModel(False) 1595 return HttpRequestMock(None, '{}', model.response)

1596

1597 1598 -class HttpMock(object):

1599 """Mock of httplib2.Http""" 1600

1601 - def __init__(self, filename=None, headers=None):

1602 """ 1603 Args: 1604 filename: string, absolute filename to read response from 1605 headers: dict, header to return with response 1606 """ 1607 if headers is None: 1608 headers = {'status': '200'} 1609 if filename: 1610 f = open(filename, 'rb') 1611 self.data = f.read() 1612 f.close() 1613 else: 1614 self.data = None 1615 self.response_headers = headers 1616 self.headers = None 1617 self.uri = None 1618 self.method = None 1619 self.body = None 1620 self.headers = None

1621 1622

1623 - def request(self, uri, 1624 method='GET', 1625 body=None, 1626 headers=None, 1627 redirections=1, 1628 connection_type=None):

1629 self.uri = uri 1630 self.method = method 1631 self.body = body 1632 self.headers = headers 1633 return httplib2.Response(self.response_headers), self.data

1634

1635 1636 -class HttpMockSequence(object):

1637 """Mock of httplib2.Http 1638 1639 Mocks a sequence of calls to request returning different responses for each 1640 call. Create an instance initialized with the desired response headers 1641 and content and then use as if an httplib2.Http instance. 1642 1643 http = HttpMockSequence([ 1644 ({'status': '401'}, ''), 1645 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1646 ({'status': '200'}, 'echo_request_headers'), 1647 ]) 1648 resp, content = http.request("http://examples.com") 1649 1650 There are special values you can pass in for content to trigger 1651 behavours that are helpful in testing. 1652 1653 'echo_request_headers' means return the request headers in the response body 1654 'echo_request_headers_as_json' means return the request headers in 1655 the response body 1656 'echo_request_body' means return the request body in the response body 1657 'echo_request_uri' means return the request uri in the response body 1658 """ 1659

1660 - def __init__(self, iterable):

1661 """ 1662 Args: 1663 iterable: iterable, a sequence of pairs of (headers, body) 1664 """ 1665 self._iterable = iterable 1666 self.follow_redirects = True

1667

1668 - def request(self, uri, 1669 method='GET', 1670 body=None, 1671 headers=None, 1672 redirections=1, 1673 connection_type=None):

1674 resp, content = self._iterable.pop(0) 1675 if content == 'echo_request_headers': 1676 content = headers 1677 elif content == 'echo_request_headers_as_json': 1678 content = json.dumps(headers) 1679 elif content == 'echo_request_body': 1680 if hasattr(body, 'read'): 1681 content = body.read() 1682 else: 1683 content = body 1684 elif content == 'echo_request_uri': 1685 content = uri 1686 if isinstance(content, six.text_type): 1687 content = content.encode('utf-8') 1688 return httplib2.Response(resp), content

1689

1690 1691 -def set_user_agent(http, user_agent):

1692 """Set the user-agent on every request. 1693 1694 Args: 1695 http - An instance of httplib2.Http 1696 or something that acts like it. 1697 user_agent: string, the value for the user-agent header. 1698 1699 Returns: 1700 A modified instance of http that was passed in. 1701 1702 Example: 1703 1704 h = httplib2.Http() 1705 h = set_user_agent(h, "my-app-name/6.0") 1706 1707 Most of the time the user-agent will be set doing auth, this is for the rare 1708 cases where you are accessing an unauthenticated endpoint. 1709 """ 1710 request_orig = http.request 1711 1712 # The closure that will replace 'httplib2.Http.request'. 1713 def new_request(uri, method='GET', body=None, headers=None, 1714 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1715 connection_type=None): 1716 """Modify the request headers to add the user-agent.""" 1717 if headers is None: 1718 headers = {} 1719 if 'user-agent' in headers: 1720 headers['user-agent'] = user_agent + ' ' + headers['user-agent'] 1721 else: 1722 headers['user-agent'] = user_agent 1723 resp, content = request_orig(uri, method=method, body=body, headers=headers, 1724 redirections=redirections, connection_type=connection_type) 1725 return resp, content

1726 1727 http.request = new_request 1728 return http 1729

1730 1731 -def tunnel_patch(http):

1732 """Tunnel PATCH requests over POST. 1733 Args: 1734 http - An instance of httplib2.Http 1735 or something that acts like it. 1736 1737 Returns: 1738 A modified instance of http that was passed in. 1739 1740 Example: 1741 1742 h = httplib2.Http() 1743 h = tunnel_patch(h, "my-app-name/6.0") 1744 1745 Useful if you are running on a platform that doesn't support PATCH. 1746 Apply this last if you are using OAuth 1.0, as changing the method 1747 will result in a different signature. 1748 """ 1749 request_orig = http.request 1750 1751 # The closure that will replace 'httplib2.Http.request'. 1752 def new_request(uri, method='GET', body=None, headers=None, 1753 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1754 connection_type=None): 1755 """Modify the request headers to add the user-agent.""" 1756 if headers is None: 1757 headers = {} 1758 if method == 'PATCH': 1759 if 'oauth_token' in headers.get('authorization', ''): 1760 LOGGER.warning( 1761 'OAuth 1.0 request made with Credentials after tunnel_patch.') 1762 headers['x-http-method-override'] = "PATCH" 1763 method = 'POST' 1764 resp, content = request_orig(uri, method=method, body=body, headers=headers, 1765 redirections=redirections, connection_type=connection_type) 1766 return resp, content

1767 1768 http.request = new_request 1769 return http 1770

1771 1772 -def build_http():

1773 """Builds httplib2.Http object 1774 1775 Returns: 1776 A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 1777 To override default timeout call 1778 1779 socket.setdefaulttimeout(timeout_in_sec) 1780 1781 before interacting with this method. 1782 """ 1783 if socket.getdefaulttimeout() is not None: 1784 http_timeout = socket.getdefaulttimeout() 1785 else: 1786 http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 1787 return httplib2.Http(timeout=http_timeout)

1788

Source Code for Module googleapiclient.http