1# Copyright 2014 Google Inc. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Classes to encapsulate a single HTTP request. 16 17The classes implement a command pattern, with every 18object supporting an execute() method that does the 19actual HTTP request. 20""" 21from __future__ import absolute_import 22import six 23from six.moves import http_client 24from six.moves import range 25 26__author__ = "jcgregorio@google.com (Joe Gregorio)" 27 28from six import BytesIO, StringIO 29from six.moves.urllib.parse import urlparse, urlunparse, quote, unquote 30 31import base64 32import copy 33import gzip 34import httplib2 35import json 36import logging 37import mimetypes 38import os 39import random 40import socket 41import sys 42import time 43import uuid 44 45# TODO(issue 221): Remove this conditional import jibbajabba. 46try: 47 import ssl 48except ImportError: 49 _ssl_SSLError = object() 50else: 51 _ssl_SSLError = ssl.SSLError 52 53from email.generator import Generator 54from email.mime.multipart import MIMEMultipart 55from email.mime.nonmultipart import MIMENonMultipart 56from email.parser import FeedParser 57 58from googleapiclient import _helpers as util 59 60from googleapiclient import _auth 61from googleapiclient.errors import BatchError 62from googleapiclient.errors import HttpError 63from googleapiclient.errors import InvalidChunkSizeError 64from googleapiclient.errors import ResumableUploadError 65from googleapiclient.errors import UnexpectedBodyError 66from googleapiclient.errors import UnexpectedMethodError 67from googleapiclient.model import JsonModel 68 69 70LOGGER = logging.getLogger(__name__) 71 72DEFAULT_CHUNK_SIZE = 100 * 1024 * 1024 73 74MAX_URI_LENGTH = 2048 75 76MAX_BATCH_LIMIT = 1000 77 78_TOO_MANY_REQUESTS = 429 79 80DEFAULT_HTTP_TIMEOUT_SEC = 60 81 82_LEGACY_BATCH_URI = "https://www.googleapis.com/batch" 83 84 85def _should_retry_response(resp_status, content): 86 """Determines whether a response should be retried. 87 88 Args: 89 resp_status: The response status received. 90 content: The response content body. 91 92 Returns: 93 True if the response should be retried, otherwise False. 94 """ 95 # Retry on 5xx errors. 96 if resp_status >= 500: 97 return True 98 99 # Retry on 429 errors. 100 if resp_status == _TOO_MANY_REQUESTS: 101 return True 102 103 # For 403 errors, we have to check for the `reason` in the response to 104 # determine if we should retry. 105 if resp_status == six.moves.http_client.FORBIDDEN: 106 # If there's no details about the 403 type, don't retry. 107 if not content: 108 return False 109 110 # Content is in JSON format. 111 try: 112 data = json.loads(content.decode("utf-8")) 113 if isinstance(data, dict): 114 reason = data["error"]["errors"][0]["reason"] 115 else: 116 reason = data[0]["error"]["errors"]["reason"] 117 except (UnicodeDecodeError, ValueError, KeyError): 118 LOGGER.warning("Invalid JSON content from response: %s", content) 119 return False 120 121 LOGGER.warning('Encountered 403 Forbidden with reason "%s"', reason) 122 123 # Only retry on rate limit related failures. 124 if reason in ("userRateLimitExceeded", "rateLimitExceeded"): 125 return True 126 127 # Everything else is a success or non-retriable so break. 128 return False 129 130 131def _retry_request( 132 http, num_retries, req_type, sleep, rand, uri, method, *args, **kwargs 133): 134 """Retries an HTTP request multiple times while handling errors. 135 136 If after all retries the request still fails, last error is either returned as 137 return value (for HTTP 5xx errors) or thrown (for ssl.SSLError). 138 139 Args: 140 http: Http object to be used to execute request. 141 num_retries: Maximum number of retries. 142 req_type: Type of the request (used for logging retries). 143 sleep, rand: Functions to sleep for random time between retries. 144 uri: URI to be requested. 145 method: HTTP method to be used. 146 args, kwargs: Additional arguments passed to http.request. 147 148 Returns: 149 resp, content - Response from the http request (may be HTTP 5xx). 150 """ 151 resp = None 152 content = None 153 exception = None 154 for retry_num in range(num_retries + 1): 155 if retry_num > 0: 156 # Sleep before retrying. 157 sleep_time = rand() * 2 ** retry_num 158 LOGGER.warning( 159 "Sleeping %.2f seconds before retry %d of %d for %s: %s %s, after %s", 160 sleep_time, 161 retry_num, 162 num_retries, 163 req_type, 164 method, 165 uri, 166 resp.status if resp else exception, 167 ) 168 sleep(sleep_time) 169 170 try: 171 exception = None 172 resp, content = http.request(uri, method, *args, **kwargs) 173 # Retry on SSL errors and socket timeout errors. 174 except _ssl_SSLError as ssl_error: 175 exception = ssl_error 176 except socket.timeout as socket_timeout: 177 # It's important that this be before socket.error as it's a subclass 178 # socket.timeout has no errorcode 179 exception = socket_timeout 180 except socket.error as socket_error: 181 # errno's contents differ by platform, so we have to match by name. 182 if socket.errno.errorcode.get(socket_error.errno) not in { 183 "WSAETIMEDOUT", 184 "ETIMEDOUT", 185 "EPIPE", 186 "ECONNABORTED", 187 }: 188 raise 189 exception = socket_error 190 except httplib2.ServerNotFoundError as server_not_found_error: 191 exception = server_not_found_error 192 193 if exception: 194 if retry_num == num_retries: 195 raise exception 196 else: 197 continue 198 199 if not _should_retry_response(resp.status, content): 200 break 201 202 return resp, content 203 204 205class MediaUploadProgress(object): 206 """Status of a resumable upload.""" 207 208 def __init__(self, resumable_progress, total_size): 209 """Constructor. 210 211 Args: 212 resumable_progress: int, bytes sent so far. 213 total_size: int, total bytes in complete upload, or None if the total 214 upload size isn't known ahead of time. 215 """ 216 self.resumable_progress = resumable_progress 217 self.total_size = total_size 218 219 def progress(self): 220 """Percent of upload completed, as a float. 221 222 Returns: 223 the percentage complete as a float, returning 0.0 if the total size of 224 the upload is unknown. 225 """ 226 if self.total_size is not None and self.total_size != 0: 227 return float(self.resumable_progress) / float(self.total_size) 228 else: 229 return 0.0 230 231 232class MediaDownloadProgress(object): 233 """Status of a resumable download.""" 234 235 def __init__(self, resumable_progress, total_size): 236 """Constructor. 237 238 Args: 239 resumable_progress: int, bytes received so far. 240 total_size: int, total bytes in complete download. 241 """ 242 self.resumable_progress = resumable_progress 243 self.total_size = total_size 244 245 def progress(self): 246 """Percent of download completed, as a float. 247 248 Returns: 249 the percentage complete as a float, returning 0.0 if the total size of 250 the download is unknown. 251 """ 252 if self.total_size is not None and self.total_size != 0: 253 return float(self.resumable_progress) / float(self.total_size) 254 else: 255 return 0.0 256 257 258class MediaUpload(object): 259 """Describes a media object to upload. 260 261 Base class that defines the interface of MediaUpload subclasses. 262 263 Note that subclasses of MediaUpload may allow you to control the chunksize 264 when uploading a media object. It is important to keep the size of the chunk 265 as large as possible to keep the upload efficient. Other factors may influence 266 the size of the chunk you use, particularly if you are working in an 267 environment where individual HTTP requests may have a hardcoded time limit, 268 such as under certain classes of requests under Google App Engine. 269 270 Streams are io.Base compatible objects that support seek(). Some MediaUpload 271 subclasses support using streams directly to upload data. Support for 272 streaming may be indicated by a MediaUpload sub-class and if appropriate for a 273 platform that stream will be used for uploading the media object. The support 274 for streaming is indicated by has_stream() returning True. The stream() method 275 should return an io.Base object that supports seek(). On platforms where the 276 underlying httplib module supports streaming, for example Python 2.6 and 277 later, the stream will be passed into the http library which will result in 278 less memory being used and possibly faster uploads. 279 280 If you need to upload media that can't be uploaded using any of the existing 281 MediaUpload sub-class then you can sub-class MediaUpload for your particular 282 needs. 283 """ 284 285 def chunksize(self): 286 """Chunk size for resumable uploads. 287 288 Returns: 289 Chunk size in bytes. 290 """ 291 raise NotImplementedError() 292 293 def mimetype(self): 294 """Mime type of the body. 295 296 Returns: 297 Mime type. 298 """ 299 return "application/octet-stream" 300 301 def size(self): 302 """Size of upload. 303 304 Returns: 305 Size of the body, or None of the size is unknown. 306 """ 307 return None 308 309 def resumable(self): 310 """Whether this upload is resumable. 311 312 Returns: 313 True if resumable upload or False. 314 """ 315 return False 316 317 def getbytes(self, begin, end): 318 """Get bytes from the media. 319 320 Args: 321 begin: int, offset from beginning of file. 322 length: int, number of bytes to read, starting at begin. 323 324 Returns: 325 A string of bytes read. May be shorter than length if EOF was reached 326 first. 327 """ 328 raise NotImplementedError() 329 330 def has_stream(self): 331 """Does the underlying upload support a streaming interface. 332 333 Streaming means it is an io.IOBase subclass that supports seek, i.e. 334 seekable() returns True. 335 336 Returns: 337 True if the call to stream() will return an instance of a seekable io.Base 338 subclass. 339 """ 340 return False 341 342 def stream(self): 343 """A stream interface to the data being uploaded. 344 345 Returns: 346 The returned value is an io.IOBase subclass that supports seek, i.e. 347 seekable() returns True. 348 """ 349 raise NotImplementedError() 350 351 @util.positional(1) 352 def _to_json(self, strip=None): 353 """Utility function for creating a JSON representation of a MediaUpload. 354 355 Args: 356 strip: array, An array of names of members to not include in the JSON. 357 358 Returns: 359 string, a JSON representation of this instance, suitable to pass to 360 from_json(). 361 """ 362 t = type(self) 363 d = copy.copy(self.__dict__) 364 if strip is not None: 365 for member in strip: 366 del d[member] 367 d["_class"] = t.__name__ 368 d["_module"] = t.__module__ 369 return json.dumps(d) 370 371 def to_json(self): 372 """Create a JSON representation of an instance of MediaUpload. 373 374 Returns: 375 string, a JSON representation of this instance, suitable to pass to 376 from_json(). 377 """ 378 return self._to_json() 379 380 @classmethod 381 def new_from_json(cls, s): 382 """Utility class method to instantiate a MediaUpload subclass from a JSON 383 representation produced by to_json(). 384 385 Args: 386 s: string, JSON from to_json(). 387 388 Returns: 389 An instance of the subclass of MediaUpload that was serialized with 390 to_json(). 391 """ 392 data = json.loads(s) 393 # Find and call the right classmethod from_json() to restore the object. 394 module = data["_module"] 395 m = __import__(module, fromlist=module.split(".")[:-1]) 396 kls = getattr(m, data["_class"]) 397 from_json = getattr(kls, "from_json") 398 return from_json(s) 399 400 401class MediaIoBaseUpload(MediaUpload): 402 """A MediaUpload for a io.Base objects. 403 404 Note that the Python file object is compatible with io.Base and can be used 405 with this class also. 406 407 fh = BytesIO('...Some data to upload...') 408 media = MediaIoBaseUpload(fh, mimetype='image/png', 409 chunksize=1024*1024, resumable=True) 410 farm.animals().insert( 411 id='cow', 412 name='cow.png', 413 media_body=media).execute() 414 415 Depending on the platform you are working on, you may pass -1 as the 416 chunksize, which indicates that the entire file should be uploaded in a single 417 request. If the underlying platform supports streams, such as Python 2.6 or 418 later, then this can be very efficient as it avoids multiple connections, and 419 also avoids loading the entire file into memory before sending it. Note that 420 Google App Engine has a 5MB limit on request size, so you should never set 421 your chunksize larger than 5MB, or to -1. 422 """ 423 424 @util.positional(3) 425 def __init__(self, fd, mimetype, chunksize=DEFAULT_CHUNK_SIZE, resumable=False): 426 """Constructor. 427 428 Args: 429 fd: io.Base or file object, The source of the bytes to upload. MUST be 430 opened in blocking mode, do not use streams opened in non-blocking mode. 431 The given stream must be seekable, that is, it must be able to call 432 seek() on fd. 433 mimetype: string, Mime-type of the file. 434 chunksize: int, File will be uploaded in chunks of this many bytes. Only 435 used if resumable=True. Pass in a value of -1 if the file is to be 436 uploaded as a single chunk. Note that Google App Engine has a 5MB limit 437 on request size, so you should never set your chunksize larger than 5MB, 438 or to -1. 439 resumable: bool, True if this is a resumable upload. False means upload 440 in a single request. 441 """ 442 super(MediaIoBaseUpload, self).__init__() 443 self._fd = fd 444 self._mimetype = mimetype 445 if not (chunksize == -1 or chunksize > 0): 446 raise InvalidChunkSizeError() 447 self._chunksize = chunksize 448 self._resumable = resumable 449 450 self._fd.seek(0, os.SEEK_END) 451 self._size = self._fd.tell() 452 453 def chunksize(self): 454 """Chunk size for resumable uploads. 455 456 Returns: 457 Chunk size in bytes. 458 """ 459 return self._chunksize 460 461 def mimetype(self): 462 """Mime type of the body. 463 464 Returns: 465 Mime type. 466 """ 467 return self._mimetype 468 469 def size(self): 470 """Size of upload. 471 472 Returns: 473 Size of the body, or None of the size is unknown. 474 """ 475 return self._size 476 477 def resumable(self): 478 """Whether this upload is resumable. 479 480 Returns: 481 True if resumable upload or False. 482 """ 483 return self._resumable 484 485 def getbytes(self, begin, length): 486 """Get bytes from the media. 487 488 Args: 489 begin: int, offset from beginning of file. 490 length: int, number of bytes to read, starting at begin. 491 492 Returns: 493 A string of bytes read. May be shorted than length if EOF was reached 494 first. 495 """ 496 self._fd.seek(begin) 497 return self._fd.read(length) 498 499 def has_stream(self): 500 """Does the underlying upload support a streaming interface. 501 502 Streaming means it is an io.IOBase subclass that supports seek, i.e. 503 seekable() returns True. 504 505 Returns: 506 True if the call to stream() will return an instance of a seekable io.Base 507 subclass. 508 """ 509 return True 510 511 def stream(self): 512 """A stream interface to the data being uploaded. 513 514 Returns: 515 The returned value is an io.IOBase subclass that supports seek, i.e. 516 seekable() returns True. 517 """ 518 return self._fd 519 520 def to_json(self): 521 """This upload type is not serializable.""" 522 raise NotImplementedError("MediaIoBaseUpload is not serializable.") 523 524 525class MediaFileUpload(MediaIoBaseUpload): 526 """A MediaUpload for a file. 527 528 Construct a MediaFileUpload and pass as the media_body parameter of the 529 method. For example, if we had a service that allowed uploading images: 530 531 media = MediaFileUpload('cow.png', mimetype='image/png', 532 chunksize=1024*1024, resumable=True) 533 farm.animals().insert( 534 id='cow', 535 name='cow.png', 536 media_body=media).execute() 537 538 Depending on the platform you are working on, you may pass -1 as the 539 chunksize, which indicates that the entire file should be uploaded in a single 540 request. If the underlying platform supports streams, such as Python 2.6 or 541 later, then this can be very efficient as it avoids multiple connections, and 542 also avoids loading the entire file into memory before sending it. Note that 543 Google App Engine has a 5MB limit on request size, so you should never set 544 your chunksize larger than 5MB, or to -1. 545 """ 546 547 @util.positional(2) 548 def __init__( 549 self, filename, mimetype=None, chunksize=DEFAULT_CHUNK_SIZE, resumable=False 550 ): 551 """Constructor. 552 553 Args: 554 filename: string, Name of the file. 555 mimetype: string, Mime-type of the file. If None then a mime-type will be 556 guessed from the file extension. 557 chunksize: int, File will be uploaded in chunks of this many bytes. Only 558 used if resumable=True. Pass in a value of -1 if the file is to be 559 uploaded in a single chunk. Note that Google App Engine has a 5MB limit 560 on request size, so you should never set your chunksize larger than 5MB, 561 or to -1. 562 resumable: bool, True if this is a resumable upload. False means upload 563 in a single request. 564 """ 565 self._filename = filename 566 fd = open(self._filename, "rb") 567 if mimetype is None: 568 # No mimetype provided, make a guess. 569 mimetype, _ = mimetypes.guess_type(filename) 570 if mimetype is None: 571 # Guess failed, use octet-stream. 572 mimetype = "application/octet-stream" 573 super(MediaFileUpload, self).__init__( 574 fd, mimetype, chunksize=chunksize, resumable=resumable 575 ) 576 577 def __del__(self): 578 self._fd.close() 579 580 def to_json(self): 581 """Creating a JSON representation of an instance of MediaFileUpload. 582 583 Returns: 584 string, a JSON representation of this instance, suitable to pass to 585 from_json(). 586 """ 587 return self._to_json(strip=["_fd"]) 588 589 @staticmethod 590 def from_json(s): 591 d = json.loads(s) 592 return MediaFileUpload( 593 d["_filename"], 594 mimetype=d["_mimetype"], 595 chunksize=d["_chunksize"], 596 resumable=d["_resumable"], 597 ) 598 599 600class MediaInMemoryUpload(MediaIoBaseUpload): 601 """MediaUpload for a chunk of bytes. 602 603 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 604 the stream. 605 """ 606 607 @util.positional(2) 608 def __init__( 609 self, 610 body, 611 mimetype="application/octet-stream", 612 chunksize=DEFAULT_CHUNK_SIZE, 613 resumable=False, 614 ): 615 """Create a new MediaInMemoryUpload. 616 617 DEPRECATED: Use MediaIoBaseUpload with either io.TextIOBase or StringIO for 618 the stream. 619 620 Args: 621 body: string, Bytes of body content. 622 mimetype: string, Mime-type of the file or default of 623 'application/octet-stream'. 624 chunksize: int, File will be uploaded in chunks of this many bytes. Only 625 used if resumable=True. 626 resumable: bool, True if this is a resumable upload. False means upload 627 in a single request. 628 """ 629 fd = BytesIO(body) 630 super(MediaInMemoryUpload, self).__init__( 631 fd, mimetype, chunksize=chunksize, resumable=resumable 632 ) 633 634 635class MediaIoBaseDownload(object): 636 """"Download media resources. 637 638 Note that the Python file object is compatible with io.Base and can be used 639 with this class also. 640 641 642 Example: 643 request = farms.animals().get_media(id='cow') 644 fh = io.FileIO('cow.png', mode='wb') 645 downloader = MediaIoBaseDownload(fh, request, chunksize=1024*1024) 646 647 done = False 648 while done is False: 649 status, done = downloader.next_chunk() 650 if status: 651 print "Download %d%%." % int(status.progress() * 100) 652 print "Download Complete!" 653 """ 654 655 @util.positional(3) 656 def __init__(self, fd, request, chunksize=DEFAULT_CHUNK_SIZE): 657 """Constructor. 658 659 Args: 660 fd: io.Base or file object, The stream in which to write the downloaded 661 bytes. 662 request: googleapiclient.http.HttpRequest, the media request to perform in 663 chunks. 664 chunksize: int, File will be downloaded in chunks of this many bytes. 665 """ 666 self._fd = fd 667 self._request = request 668 self._uri = request.uri 669 self._chunksize = chunksize 670 self._progress = 0 671 self._total_size = None 672 self._done = False 673 674 # Stubs for testing. 675 self._sleep = time.sleep 676 self._rand = random.random 677 678 self._headers = {} 679 for k, v in six.iteritems(request.headers): 680 # allow users to supply custom headers by setting them on the request 681 # but strip out the ones that are set by default on requests generated by 682 # API methods like Drive's files().get(fileId=...) 683 if not k.lower() in ("accept", "accept-encoding", "user-agent"): 684 self._headers[k] = v 685 686 @util.positional(1) 687 def next_chunk(self, num_retries=0): 688 """Get the next chunk of the download. 689 690 Args: 691 num_retries: Integer, number of times to retry with randomized 692 exponential backoff. If all retries fail, the raised HttpError 693 represents the last request. If zero (default), we attempt the 694 request only once. 695 696 Returns: 697 (status, done): (MediaDownloadProgress, boolean) 698 The value of 'done' will be True when the media has been fully 699 downloaded or the total size of the media is unknown. 700 701 Raises: 702 googleapiclient.errors.HttpError if the response was not a 2xx. 703 httplib2.HttpLib2Error if a transport error has occured. 704 """ 705 headers = self._headers.copy() 706 headers["range"] = "bytes=%d-%d" % ( 707 self._progress, 708 self._progress + self._chunksize, 709 ) 710 http = self._request.http 711 712 resp, content = _retry_request( 713 http, 714 num_retries, 715 "media download", 716 self._sleep, 717 self._rand, 718 self._uri, 719 "GET", 720 headers=headers, 721 ) 722 723 if resp.status in [200, 206]: 724 if "content-location" in resp and resp["content-location"] != self._uri: 725 self._uri = resp["content-location"] 726 self._progress += len(content) 727 self._fd.write(content) 728 729 if "content-range" in resp: 730 content_range = resp["content-range"] 731 length = content_range.rsplit("/", 1)[1] 732 self._total_size = int(length) 733 elif "content-length" in resp: 734 self._total_size = int(resp["content-length"]) 735 736 if self._total_size is None or self._progress == self._total_size: 737 self._done = True 738 return MediaDownloadProgress(self._progress, self._total_size), self._done 739 else: 740 raise HttpError(resp, content, uri=self._uri) 741 742 743class _StreamSlice(object): 744 """Truncated stream. 745 746 Takes a stream and presents a stream that is a slice of the original stream. 747 This is used when uploading media in chunks. In later versions of Python a 748 stream can be passed to httplib in place of the string of data to send. The 749 problem is that httplib just blindly reads to the end of the stream. This 750 wrapper presents a virtual stream that only reads to the end of the chunk. 751 """ 752 753 def __init__(self, stream, begin, chunksize): 754 """Constructor. 755 756 Args: 757 stream: (io.Base, file object), the stream to wrap. 758 begin: int, the seek position the chunk begins at. 759 chunksize: int, the size of the chunk. 760 """ 761 self._stream = stream 762 self._begin = begin 763 self._chunksize = chunksize 764 self._stream.seek(begin) 765 766 def read(self, n=-1): 767 """Read n bytes. 768 769 Args: 770 n, int, the number of bytes to read. 771 772 Returns: 773 A string of length 'n', or less if EOF is reached. 774 """ 775 # The data left available to read sits in [cur, end) 776 cur = self._stream.tell() 777 end = self._begin + self._chunksize 778 if n == -1 or cur + n > end: 779 n = end - cur 780 return self._stream.read(n) 781 782 783class HttpRequest(object): 784 """Encapsulates a single HTTP request.""" 785 786 @util.positional(4) 787 def __init__( 788 self, 789 http, 790 postproc, 791 uri, 792 method="GET", 793 body=None, 794 headers=None, 795 methodId=None, 796 resumable=None, 797 ): 798 """Constructor for an HttpRequest. 799 800 Args: 801 http: httplib2.Http, the transport object to use to make a request 802 postproc: callable, called on the HTTP response and content to transform 803 it into a data object before returning, or raising an exception 804 on an error. 805 uri: string, the absolute URI to send the request to 806 method: string, the HTTP method to use 807 body: string, the request body of the HTTP request, 808 headers: dict, the HTTP request headers 809 methodId: string, a unique identifier for the API method being called. 810 resumable: MediaUpload, None if this is not a resumbale request. 811 """ 812 self.uri = uri 813 self.method = method 814 self.body = body 815 self.headers = headers or {} 816 self.methodId = methodId 817 self.http = http 818 self.postproc = postproc 819 self.resumable = resumable 820 self.response_callbacks = [] 821 self._in_error_state = False 822 823 # The size of the non-media part of the request. 824 self.body_size = len(self.body or "") 825 826 # The resumable URI to send chunks to. 827 self.resumable_uri = None 828 829 # The bytes that have been uploaded. 830 self.resumable_progress = 0 831 832 # Stubs for testing. 833 self._rand = random.random 834 self._sleep = time.sleep 835 836 @util.positional(1) 837 def execute(self, http=None, num_retries=0): 838 """Execute the request. 839 840 Args: 841 http: httplib2.Http, an http object to be used in place of the 842 one the HttpRequest request object was constructed with. 843 num_retries: Integer, number of times to retry with randomized 844 exponential backoff. If all retries fail, the raised HttpError 845 represents the last request. If zero (default), we attempt the 846 request only once. 847 848 Returns: 849 A deserialized object model of the response body as determined 850 by the postproc. 851 852 Raises: 853 googleapiclient.errors.HttpError if the response was not a 2xx. 854 httplib2.HttpLib2Error if a transport error has occured. 855 """ 856 if http is None: 857 http = self.http 858 859 if self.resumable: 860 body = None 861 while body is None: 862 _, body = self.next_chunk(http=http, num_retries=num_retries) 863 return body 864 865 # Non-resumable case. 866 867 if "content-length" not in self.headers: 868 self.headers["content-length"] = str(self.body_size) 869 # If the request URI is too long then turn it into a POST request. 870 # Assume that a GET request never contains a request body. 871 if len(self.uri) > MAX_URI_LENGTH and self.method == "GET": 872 self.method = "POST" 873 self.headers["x-http-method-override"] = "GET" 874 self.headers["content-type"] = "application/x-www-form-urlencoded" 875 parsed = urlparse(self.uri) 876 self.uri = urlunparse( 877 (parsed.scheme, parsed.netloc, parsed.path, parsed.params, None, None) 878 ) 879 self.body = parsed.query 880 self.headers["content-length"] = str(len(self.body)) 881 882 # Handle retries for server-side errors. 883 resp, content = _retry_request( 884 http, 885 num_retries, 886 "request", 887 self._sleep, 888 self._rand, 889 str(self.uri), 890 method=str(self.method), 891 body=self.body, 892 headers=self.headers, 893 ) 894 895 for callback in self.response_callbacks: 896 callback(resp) 897 if resp.status >= 300: 898 raise HttpError(resp, content, uri=self.uri) 899 return self.postproc(resp, content) 900 901 @util.positional(2) 902 def add_response_callback(self, cb): 903 """add_response_headers_callback 904 905 Args: 906 cb: Callback to be called on receiving the response headers, of signature: 907 908 def cb(resp): 909 # Where resp is an instance of httplib2.Response 910 """ 911 self.response_callbacks.append(cb) 912 913 @util.positional(1) 914 def next_chunk(self, http=None, num_retries=0): 915 """Execute the next step of a resumable upload. 916 917 Can only be used if the method being executed supports media uploads and 918 the MediaUpload object passed in was flagged as using resumable upload. 919 920 Example: 921 922 media = MediaFileUpload('cow.png', mimetype='image/png', 923 chunksize=1000, resumable=True) 924 request = farm.animals().insert( 925 id='cow', 926 name='cow.png', 927 media_body=media) 928 929 response = None 930 while response is None: 931 status, response = request.next_chunk() 932 if status: 933 print "Upload %d%% complete." % int(status.progress() * 100) 934 935 936 Args: 937 http: httplib2.Http, an http object to be used in place of the 938 one the HttpRequest request object was constructed with. 939 num_retries: Integer, number of times to retry with randomized 940 exponential backoff. If all retries fail, the raised HttpError 941 represents the last request. If zero (default), we attempt the 942 request only once. 943 944 Returns: 945 (status, body): (ResumableMediaStatus, object) 946 The body will be None until the resumable media is fully uploaded. 947 948 Raises: 949 googleapiclient.errors.HttpError if the response was not a 2xx. 950 httplib2.HttpLib2Error if a transport error has occured. 951 """ 952 if http is None: 953 http = self.http 954 955 if self.resumable.size() is None: 956 size = "*" 957 else: 958 size = str(self.resumable.size()) 959 960 if self.resumable_uri is None: 961 start_headers = copy.copy(self.headers) 962 start_headers["X-Upload-Content-Type"] = self.resumable.mimetype() 963 if size != "*": 964 start_headers["X-Upload-Content-Length"] = size 965 start_headers["content-length"] = str(self.body_size) 966 967 resp, content = _retry_request( 968 http, 969 num_retries, 970 "resumable URI request", 971 self._sleep, 972 self._rand, 973 self.uri, 974 method=self.method, 975 body=self.body, 976 headers=start_headers, 977 ) 978 979 if resp.status == 200 and "location" in resp: 980 self.resumable_uri = resp["location"] 981 else: 982 raise ResumableUploadError(resp, content) 983 elif self._in_error_state: 984 # If we are in an error state then query the server for current state of 985 # the upload by sending an empty PUT and reading the 'range' header in 986 # the response. 987 headers = {"Content-Range": "bytes */%s" % size, "content-length": "0"} 988 resp, content = http.request(self.resumable_uri, "PUT", headers=headers) 989 status, body = self._process_response(resp, content) 990 if body: 991 # The upload was complete. 992 return (status, body) 993 994 if self.resumable.has_stream(): 995 data = self.resumable.stream() 996 if self.resumable.chunksize() == -1: 997 data.seek(self.resumable_progress) 998 chunk_end = self.resumable.size() - self.resumable_progress - 1 999 else: 1000 # Doing chunking with a stream, so wrap a slice of the stream. 1001 data = _StreamSlice( 1002 data, self.resumable_progress, self.resumable.chunksize() 1003 ) 1004 chunk_end = min( 1005 self.resumable_progress + self.resumable.chunksize() - 1, 1006 self.resumable.size() - 1, 1007 ) 1008 else: 1009 data = self.resumable.getbytes( 1010 self.resumable_progress, self.resumable.chunksize() 1011 ) 1012 1013 # A short read implies that we are at EOF, so finish the upload. 1014 if len(data) < self.resumable.chunksize(): 1015 size = str(self.resumable_progress + len(data)) 1016 1017 chunk_end = self.resumable_progress + len(data) - 1 1018 1019 headers = { 1020 "Content-Range": "bytes %d-%d/%s" 1021 % (self.resumable_progress, chunk_end, size), 1022 # Must set the content-length header here because httplib can't 1023 # calculate the size when working with _StreamSlice. 1024 "Content-Length": str(chunk_end - self.resumable_progress + 1), 1025 } 1026 1027 for retry_num in range(num_retries + 1): 1028 if retry_num > 0: 1029 self._sleep(self._rand() * 2 ** retry_num) 1030 LOGGER.warning( 1031 "Retry #%d for media upload: %s %s, following status: %d" 1032 % (retry_num, self.method, self.uri, resp.status) 1033 ) 1034 1035 try: 1036 resp, content = http.request( 1037 self.resumable_uri, method="PUT", body=data, headers=headers 1038 ) 1039 except: 1040 self._in_error_state = True 1041 raise 1042 if not _should_retry_response(resp.status, content): 1043 break 1044 1045 return self._process_response(resp, content) 1046 1047 def _process_response(self, resp, content): 1048 """Process the response from a single chunk upload. 1049 1050 Args: 1051 resp: httplib2.Response, the response object. 1052 content: string, the content of the response. 1053 1054 Returns: 1055 (status, body): (ResumableMediaStatus, object) 1056 The body will be None until the resumable media is fully uploaded. 1057 1058 Raises: 1059 googleapiclient.errors.HttpError if the response was not a 2xx or a 308. 1060 """ 1061 if resp.status in [200, 201]: 1062 self._in_error_state = False 1063 return None, self.postproc(resp, content) 1064 elif resp.status == 308: 1065 self._in_error_state = False 1066 # A "308 Resume Incomplete" indicates we are not done. 1067 try: 1068 self.resumable_progress = int(resp["range"].split("-")[1]) + 1 1069 except KeyError: 1070 # If resp doesn't contain range header, resumable progress is 0 1071 self.resumable_progress = 0 1072 if "location" in resp: 1073 self.resumable_uri = resp["location"] 1074 else: 1075 self._in_error_state = True 1076 raise HttpError(resp, content, uri=self.uri) 1077 1078 return ( 1079 MediaUploadProgress(self.resumable_progress, self.resumable.size()), 1080 None, 1081 ) 1082 1083 def to_json(self): 1084 """Returns a JSON representation of the HttpRequest.""" 1085 d = copy.copy(self.__dict__) 1086 if d["resumable"] is not None: 1087 d["resumable"] = self.resumable.to_json() 1088 del d["http"] 1089 del d["postproc"] 1090 del d["_sleep"] 1091 del d["_rand"] 1092 1093 return json.dumps(d) 1094 1095 @staticmethod 1096 def from_json(s, http, postproc): 1097 """Returns an HttpRequest populated with info from a JSON object.""" 1098 d = json.loads(s) 1099 if d["resumable"] is not None: 1100 d["resumable"] = MediaUpload.new_from_json(d["resumable"]) 1101 return HttpRequest( 1102 http, 1103 postproc, 1104 uri=d["uri"], 1105 method=d["method"], 1106 body=d["body"], 1107 headers=d["headers"], 1108 methodId=d["methodId"], 1109 resumable=d["resumable"], 1110 ) 1111 1112 1113class BatchHttpRequest(object): 1114 """Batches multiple HttpRequest objects into a single HTTP request. 1115 1116 Example: 1117 from googleapiclient.http import BatchHttpRequest 1118 1119 def list_animals(request_id, response, exception): 1120 \"\"\"Do something with the animals list response.\"\"\" 1121 if exception is not None: 1122 # Do something with the exception. 1123 pass 1124 else: 1125 # Do something with the response. 1126 pass 1127 1128 def list_farmers(request_id, response, exception): 1129 \"\"\"Do something with the farmers list response.\"\"\" 1130 if exception is not None: 1131 # Do something with the exception. 1132 pass 1133 else: 1134 # Do something with the response. 1135 pass 1136 1137 service = build('farm', 'v2') 1138 1139 batch = BatchHttpRequest() 1140 1141 batch.add(service.animals().list(), list_animals) 1142 batch.add(service.farmers().list(), list_farmers) 1143 batch.execute(http=http) 1144 """ 1145 1146 @util.positional(1) 1147 def __init__(self, callback=None, batch_uri=None): 1148 """Constructor for a BatchHttpRequest. 1149 1150 Args: 1151 callback: callable, A callback to be called for each response, of the 1152 form callback(id, response, exception). The first parameter is the 1153 request id, and the second is the deserialized response object. The 1154 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1155 occurred while processing the request, or None if no error occurred. 1156 batch_uri: string, URI to send batch requests to. 1157 """ 1158 if batch_uri is None: 1159 batch_uri = _LEGACY_BATCH_URI 1160 1161 if batch_uri == _LEGACY_BATCH_URI: 1162 LOGGER.warn( 1163 "You have constructed a BatchHttpRequest using the legacy batch " 1164 "endpoint %s. This endpoint will be turned down on March 25, 2019. " 1165 "Please provide the API-specific endpoint or use " 1166 "service.new_batch_http_request(). For more details see " 1167 "https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html" 1168 "and https://developers.google.com/api-client-library/python/guide/batch.", 1169 _LEGACY_BATCH_URI, 1170 ) 1171 self._batch_uri = batch_uri 1172 1173 # Global callback to be called for each individual response in the batch. 1174 self._callback = callback 1175 1176 # A map from id to request. 1177 self._requests = {} 1178 1179 # A map from id to callback. 1180 self._callbacks = {} 1181 1182 # List of request ids, in the order in which they were added. 1183 self._order = [] 1184 1185 # The last auto generated id. 1186 self._last_auto_id = 0 1187 1188 # Unique ID on which to base the Content-ID headers. 1189 self._base_id = None 1190 1191 # A map from request id to (httplib2.Response, content) response pairs 1192 self._responses = {} 1193 1194 # A map of id(Credentials) that have been refreshed. 1195 self._refreshed_credentials = {} 1196 1197 def _refresh_and_apply_credentials(self, request, http): 1198 """Refresh the credentials and apply to the request. 1199 1200 Args: 1201 request: HttpRequest, the request. 1202 http: httplib2.Http, the global http object for the batch. 1203 """ 1204 # For the credentials to refresh, but only once per refresh_token 1205 # If there is no http per the request then refresh the http passed in 1206 # via execute() 1207 creds = None 1208 request_credentials = False 1209 1210 if request.http is not None: 1211 creds = _auth.get_credentials_from_http(request.http) 1212 request_credentials = True 1213 1214 if creds is None and http is not None: 1215 creds = _auth.get_credentials_from_http(http) 1216 1217 if creds is not None: 1218 if id(creds) not in self._refreshed_credentials: 1219 _auth.refresh_credentials(creds) 1220 self._refreshed_credentials[id(creds)] = 1 1221 1222 # Only apply the credentials if we are using the http object passed in, 1223 # otherwise apply() will get called during _serialize_request(). 1224 if request.http is None or not request_credentials: 1225 _auth.apply_credentials(creds, request.headers) 1226 1227 def _id_to_header(self, id_): 1228 """Convert an id to a Content-ID header value. 1229 1230 Args: 1231 id_: string, identifier of individual request. 1232 1233 Returns: 1234 A Content-ID header with the id_ encoded into it. A UUID is prepended to 1235 the value because Content-ID headers are supposed to be universally 1236 unique. 1237 """ 1238 if self._base_id is None: 1239 self._base_id = uuid.uuid4() 1240 1241 # NB: we intentionally leave whitespace between base/id and '+', so RFC2822 1242 # line folding works properly on Python 3; see 1243 # https://github.com/google/google-api-python-client/issues/164 1244 return "<%s + %s>" % (self._base_id, quote(id_)) 1245 1246 def _header_to_id(self, header): 1247 """Convert a Content-ID header value to an id. 1248 1249 Presumes the Content-ID header conforms to the format that _id_to_header() 1250 returns. 1251 1252 Args: 1253 header: string, Content-ID header value. 1254 1255 Returns: 1256 The extracted id value. 1257 1258 Raises: 1259 BatchError if the header is not in the expected format. 1260 """ 1261 if header[0] != "<" or header[-1] != ">": 1262 raise BatchError("Invalid value for Content-ID: %s" % header) 1263 if "+" not in header: 1264 raise BatchError("Invalid value for Content-ID: %s" % header) 1265 base, id_ = header[1:-1].split(" + ", 1) 1266 1267 return unquote(id_) 1268 1269 def _serialize_request(self, request): 1270 """Convert an HttpRequest object into a string. 1271 1272 Args: 1273 request: HttpRequest, the request to serialize. 1274 1275 Returns: 1276 The request as a string in application/http format. 1277 """ 1278 # Construct status line 1279 parsed = urlparse(request.uri) 1280 request_line = urlunparse( 1281 ("", "", parsed.path, parsed.params, parsed.query, "") 1282 ) 1283 status_line = request.method + " " + request_line + " HTTP/1.1\n" 1284 major, minor = request.headers.get("content-type", "application/json").split( 1285 "/" 1286 ) 1287 msg = MIMENonMultipart(major, minor) 1288 headers = request.headers.copy() 1289 1290 if request.http is not None: 1291 credentials = _auth.get_credentials_from_http(request.http) 1292 if credentials is not None: 1293 _auth.apply_credentials(credentials, headers) 1294 1295 # MIMENonMultipart adds its own Content-Type header. 1296 if "content-type" in headers: 1297 del headers["content-type"] 1298 1299 for key, value in six.iteritems(headers): 1300 msg[key] = value 1301 msg["Host"] = parsed.netloc 1302 msg.set_unixfrom(None) 1303 1304 if request.body is not None: 1305 msg.set_payload(request.body) 1306 msg["content-length"] = str(len(request.body)) 1307 1308 # Serialize the mime message. 1309 fp = StringIO() 1310 # maxheaderlen=0 means don't line wrap headers. 1311 g = Generator(fp, maxheaderlen=0) 1312 g.flatten(msg, unixfrom=False) 1313 body = fp.getvalue() 1314 1315 return status_line + body 1316 1317 def _deserialize_response(self, payload): 1318 """Convert string into httplib2 response and content. 1319 1320 Args: 1321 payload: string, headers and body as a string. 1322 1323 Returns: 1324 A pair (resp, content), such as would be returned from httplib2.request. 1325 """ 1326 # Strip off the status line 1327 status_line, payload = payload.split("\n", 1) 1328 protocol, status, reason = status_line.split(" ", 2) 1329 1330 # Parse the rest of the response 1331 parser = FeedParser() 1332 parser.feed(payload) 1333 msg = parser.close() 1334 msg["status"] = status 1335 1336 # Create httplib2.Response from the parsed headers. 1337 resp = httplib2.Response(msg) 1338 resp.reason = reason 1339 resp.version = int(protocol.split("/", 1)[1].replace(".", "")) 1340 1341 content = payload.split("\r\n\r\n", 1)[1] 1342 1343 return resp, content 1344 1345 def _new_id(self): 1346 """Create a new id. 1347 1348 Auto incrementing number that avoids conflicts with ids already used. 1349 1350 Returns: 1351 string, a new unique id. 1352 """ 1353 self._last_auto_id += 1 1354 while str(self._last_auto_id) in self._requests: 1355 self._last_auto_id += 1 1356 return str(self._last_auto_id) 1357 1358 @util.positional(2) 1359 def add(self, request, callback=None, request_id=None): 1360 """Add a new request. 1361 1362 Every callback added will be paired with a unique id, the request_id. That 1363 unique id will be passed back to the callback when the response comes back 1364 from the server. The default behavior is to have the library generate it's 1365 own unique id. If the caller passes in a request_id then they must ensure 1366 uniqueness for each request_id, and if they are not an exception is 1367 raised. Callers should either supply all request_ids or never supply a 1368 request id, to avoid such an error. 1369 1370 Args: 1371 request: HttpRequest, Request to add to the batch. 1372 callback: callable, A callback to be called for this response, of the 1373 form callback(id, response, exception). The first parameter is the 1374 request id, and the second is the deserialized response object. The 1375 third is an googleapiclient.errors.HttpError exception object if an HTTP error 1376 occurred while processing the request, or None if no errors occurred. 1377 request_id: string, A unique id for the request. The id will be passed 1378 to the callback with the response. 1379 1380 Returns: 1381 None 1382 1383 Raises: 1384 BatchError if a media request is added to a batch. 1385 KeyError is the request_id is not unique. 1386 """ 1387 1388 if len(self._order) >= MAX_BATCH_LIMIT: 1389 raise BatchError( 1390 "Exceeded the maximum calls(%d) in a single batch request." 1391 % MAX_BATCH_LIMIT 1392 ) 1393 if request_id is None: 1394 request_id = self._new_id() 1395 if request.resumable is not None: 1396 raise BatchError("Media requests cannot be used in a batch request.") 1397 if request_id in self._requests: 1398 raise KeyError("A request with this ID already exists: %s" % request_id) 1399 self._requests[request_id] = request 1400 self._callbacks[request_id] = callback 1401 self._order.append(request_id) 1402 1403 def _execute(self, http, order, requests): 1404 """Serialize batch request, send to server, process response. 1405 1406 Args: 1407 http: httplib2.Http, an http object to be used to make the request with. 1408 order: list, list of request ids in the order they were added to the 1409 batch. 1410 request: list, list of request objects to send. 1411 1412 Raises: 1413 httplib2.HttpLib2Error if a transport error has occured. 1414 googleapiclient.errors.BatchError if the response is the wrong format. 1415 """ 1416 message = MIMEMultipart("mixed") 1417 # Message should not write out it's own headers. 1418 setattr(message, "_write_headers", lambda self: None) 1419 1420 # Add all the individual requests. 1421 for request_id in order: 1422 request = requests[request_id] 1423 1424 msg = MIMENonMultipart("application", "http") 1425 msg["Content-Transfer-Encoding"] = "binary" 1426 msg["Content-ID"] = self._id_to_header(request_id) 1427 1428 body = self._serialize_request(request) 1429 msg.set_payload(body) 1430 message.attach(msg) 1431 1432 # encode the body: note that we can't use `as_string`, because 1433 # it plays games with `From ` lines. 1434 fp = StringIO() 1435 g = Generator(fp, mangle_from_=False) 1436 g.flatten(message, unixfrom=False) 1437 body = fp.getvalue() 1438 1439 headers = {} 1440 headers["content-type"] = ( 1441 "multipart/mixed; " 'boundary="%s"' 1442 ) % message.get_boundary() 1443 1444 resp, content = http.request( 1445 self._batch_uri, method="POST", body=body, headers=headers 1446 ) 1447 1448 if resp.status >= 300: 1449 raise HttpError(resp, content, uri=self._batch_uri) 1450 1451 # Prepend with a content-type header so FeedParser can handle it. 1452 header = "content-type: %s\r\n\r\n" % resp["content-type"] 1453 # PY3's FeedParser only accepts unicode. So we should decode content 1454 # here, and encode each payload again. 1455 if six.PY3: 1456 content = content.decode("utf-8") 1457 for_parser = header + content 1458 1459 parser = FeedParser() 1460 parser.feed(for_parser) 1461 mime_response = parser.close() 1462 1463 if not mime_response.is_multipart(): 1464 raise BatchError( 1465 "Response not in multipart/mixed format.", resp=resp, content=content 1466 ) 1467 1468 for part in mime_response.get_payload(): 1469 request_id = self._header_to_id(part["Content-ID"]) 1470 response, content = self._deserialize_response(part.get_payload()) 1471 # We encode content here to emulate normal http response. 1472 if isinstance(content, six.text_type): 1473 content = content.encode("utf-8") 1474 self._responses[request_id] = (response, content) 1475 1476 @util.positional(1) 1477 def execute(self, http=None): 1478 """Execute all the requests as a single batched HTTP request. 1479 1480 Args: 1481 http: httplib2.Http, an http object to be used in place of the one the 1482 HttpRequest request object was constructed with. If one isn't supplied 1483 then use a http object from the requests in this batch. 1484 1485 Returns: 1486 None 1487 1488 Raises: 1489 httplib2.HttpLib2Error if a transport error has occured. 1490 googleapiclient.errors.BatchError if the response is the wrong format. 1491 """ 1492 # If we have no requests return 1493 if len(self._order) == 0: 1494 return None 1495 1496 # If http is not supplied use the first valid one given in the requests. 1497 if http is None: 1498 for request_id in self._order: 1499 request = self._requests[request_id] 1500 if request is not None: 1501 http = request.http 1502 break 1503 1504 if http is None: 1505 raise ValueError("Missing a valid http object.") 1506 1507 # Special case for OAuth2Credentials-style objects which have not yet been 1508 # refreshed with an initial access_token. 1509 creds = _auth.get_credentials_from_http(http) 1510 if creds is not None: 1511 if not _auth.is_valid(creds): 1512 LOGGER.info("Attempting refresh to obtain initial access_token") 1513 _auth.refresh_credentials(creds) 1514 1515 self._execute(http, self._order, self._requests) 1516 1517 # Loop over all the requests and check for 401s. For each 401 request the 1518 # credentials should be refreshed and then sent again in a separate batch. 1519 redo_requests = {} 1520 redo_order = [] 1521 1522 for request_id in self._order: 1523 resp, content = self._responses[request_id] 1524 if resp["status"] == "401": 1525 redo_order.append(request_id) 1526 request = self._requests[request_id] 1527 self._refresh_and_apply_credentials(request, http) 1528 redo_requests[request_id] = request 1529 1530 if redo_requests: 1531 self._execute(http, redo_order, redo_requests) 1532 1533 # Now process all callbacks that are erroring, and raise an exception for 1534 # ones that return a non-2xx response? Or add extra parameter to callback 1535 # that contains an HttpError? 1536 1537 for request_id in self._order: 1538 resp, content = self._responses[request_id] 1539 1540 request = self._requests[request_id] 1541 callback = self._callbacks[request_id] 1542 1543 response = None 1544 exception = None 1545 try: 1546 if resp.status >= 300: 1547 raise HttpError(resp, content, uri=request.uri) 1548 response = request.postproc(resp, content) 1549 except HttpError as e: 1550 exception = e 1551 1552 if callback is not None: 1553 callback(request_id, response, exception) 1554 if self._callback is not None: 1555 self._callback(request_id, response, exception) 1556 1557 1558class HttpRequestMock(object): 1559 """Mock of HttpRequest. 1560 1561 Do not construct directly, instead use RequestMockBuilder. 1562 """ 1563 1564 def __init__(self, resp, content, postproc): 1565 """Constructor for HttpRequestMock 1566 1567 Args: 1568 resp: httplib2.Response, the response to emulate coming from the request 1569 content: string, the response body 1570 postproc: callable, the post processing function usually supplied by 1571 the model class. See model.JsonModel.response() as an example. 1572 """ 1573 self.resp = resp 1574 self.content = content 1575 self.postproc = postproc 1576 if resp is None: 1577 self.resp = httplib2.Response({"status": 200, "reason": "OK"}) 1578 if "reason" in self.resp: 1579 self.resp.reason = self.resp["reason"] 1580 1581 def execute(self, http=None): 1582 """Execute the request. 1583 1584 Same behavior as HttpRequest.execute(), but the response is 1585 mocked and not really from an HTTP request/response. 1586 """ 1587 return self.postproc(self.resp, self.content) 1588 1589 1590class RequestMockBuilder(object): 1591 """A simple mock of HttpRequest 1592 1593 Pass in a dictionary to the constructor that maps request methodIds to 1594 tuples of (httplib2.Response, content, opt_expected_body) that should be 1595 returned when that method is called. None may also be passed in for the 1596 httplib2.Response, in which case a 200 OK response will be generated. 1597 If an opt_expected_body (str or dict) is provided, it will be compared to 1598 the body and UnexpectedBodyError will be raised on inequality. 1599 1600 Example: 1601 response = '{"data": {"id": "tag:google.c...' 1602 requestBuilder = RequestMockBuilder( 1603 { 1604 'plus.activities.get': (None, response), 1605 } 1606 ) 1607 googleapiclient.discovery.build("plus", "v1", requestBuilder=requestBuilder) 1608 1609 Methods that you do not supply a response for will return a 1610 200 OK with an empty string as the response content or raise an excpetion 1611 if check_unexpected is set to True. The methodId is taken from the rpcName 1612 in the discovery document. 1613 1614 For more details see the project wiki. 1615 """ 1616 1617 def __init__(self, responses, check_unexpected=False): 1618 """Constructor for RequestMockBuilder 1619 1620 The constructed object should be a callable object 1621 that can replace the class HttpResponse. 1622 1623 responses - A dictionary that maps methodIds into tuples 1624 of (httplib2.Response, content). The methodId 1625 comes from the 'rpcName' field in the discovery 1626 document. 1627 check_unexpected - A boolean setting whether or not UnexpectedMethodError 1628 should be raised on unsupplied method. 1629 """ 1630 self.responses = responses 1631 self.check_unexpected = check_unexpected 1632 1633 def __call__( 1634 self, 1635 http, 1636 postproc, 1637 uri, 1638 method="GET", 1639 body=None, 1640 headers=None, 1641 methodId=None, 1642 resumable=None, 1643 ): 1644 """Implements the callable interface that discovery.build() expects 1645 of requestBuilder, which is to build an object compatible with 1646 HttpRequest.execute(). See that method for the description of the 1647 parameters and the expected response. 1648 """ 1649 if methodId in self.responses: 1650 response = self.responses[methodId] 1651 resp, content = response[:2] 1652 if len(response) > 2: 1653 # Test the body against the supplied expected_body. 1654 expected_body = response[2] 1655 if bool(expected_body) != bool(body): 1656 # Not expecting a body and provided one 1657 # or expecting a body and not provided one. 1658 raise UnexpectedBodyError(expected_body, body) 1659 if isinstance(expected_body, str): 1660 expected_body = json.loads(expected_body) 1661 body = json.loads(body) 1662 if body != expected_body: 1663 raise UnexpectedBodyError(expected_body, body) 1664 return HttpRequestMock(resp, content, postproc) 1665 elif self.check_unexpected: 1666 raise UnexpectedMethodError(methodId=methodId) 1667 else: 1668 model = JsonModel(False) 1669 return HttpRequestMock(None, "{}", model.response) 1670 1671 1672class HttpMock(object): 1673 """Mock of httplib2.Http""" 1674 1675 def __init__(self, filename=None, headers=None): 1676 """ 1677 Args: 1678 filename: string, absolute filename to read response from 1679 headers: dict, header to return with response 1680 """ 1681 if headers is None: 1682 headers = {"status": "200"} 1683 if filename: 1684 f = open(filename, "rb") 1685 self.data = f.read() 1686 f.close() 1687 else: 1688 self.data = None 1689 self.response_headers = headers 1690 self.headers = None 1691 self.uri = None 1692 self.method = None 1693 self.body = None 1694 self.headers = None 1695 1696 def request( 1697 self, 1698 uri, 1699 method="GET", 1700 body=None, 1701 headers=None, 1702 redirections=1, 1703 connection_type=None, 1704 ): 1705 self.uri = uri 1706 self.method = method 1707 self.body = body 1708 self.headers = headers 1709 return httplib2.Response(self.response_headers), self.data 1710 1711 1712class HttpMockSequence(object): 1713 """Mock of httplib2.Http 1714 1715 Mocks a sequence of calls to request returning different responses for each 1716 call. Create an instance initialized with the desired response headers 1717 and content and then use as if an httplib2.Http instance. 1718 1719 http = HttpMockSequence([ 1720 ({'status': '401'}, ''), 1721 ({'status': '200'}, '{"access_token":"1/3w","expires_in":3600}'), 1722 ({'status': '200'}, 'echo_request_headers'), 1723 ]) 1724 resp, content = http.request("http://examples.com") 1725 1726 There are special values you can pass in for content to trigger 1727 behavours that are helpful in testing. 1728 1729 'echo_request_headers' means return the request headers in the response body 1730 'echo_request_headers_as_json' means return the request headers in 1731 the response body 1732 'echo_request_body' means return the request body in the response body 1733 'echo_request_uri' means return the request uri in the response body 1734 """ 1735 1736 def __init__(self, iterable): 1737 """ 1738 Args: 1739 iterable: iterable, a sequence of pairs of (headers, body) 1740 """ 1741 self._iterable = iterable 1742 self.follow_redirects = True 1743 1744 def request( 1745 self, 1746 uri, 1747 method="GET", 1748 body=None, 1749 headers=None, 1750 redirections=1, 1751 connection_type=None, 1752 ): 1753 resp, content = self._iterable.pop(0) 1754 if content == "echo_request_headers": 1755 content = headers 1756 elif content == "echo_request_headers_as_json": 1757 content = json.dumps(headers) 1758 elif content == "echo_request_body": 1759 if hasattr(body, "read"): 1760 content = body.read() 1761 else: 1762 content = body 1763 elif content == "echo_request_uri": 1764 content = uri 1765 if isinstance(content, six.text_type): 1766 content = content.encode("utf-8") 1767 return httplib2.Response(resp), content 1768 1769 1770def set_user_agent(http, user_agent): 1771 """Set the user-agent on every request. 1772 1773 Args: 1774 http - An instance of httplib2.Http 1775 or something that acts like it. 1776 user_agent: string, the value for the user-agent header. 1777 1778 Returns: 1779 A modified instance of http that was passed in. 1780 1781 Example: 1782 1783 h = httplib2.Http() 1784 h = set_user_agent(h, "my-app-name/6.0") 1785 1786 Most of the time the user-agent will be set doing auth, this is for the rare 1787 cases where you are accessing an unauthenticated endpoint. 1788 """ 1789 request_orig = http.request 1790 1791 # The closure that will replace 'httplib2.Http.request'. 1792 def new_request( 1793 uri, 1794 method="GET", 1795 body=None, 1796 headers=None, 1797 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1798 connection_type=None, 1799 ): 1800 """Modify the request headers to add the user-agent.""" 1801 if headers is None: 1802 headers = {} 1803 if "user-agent" in headers: 1804 headers["user-agent"] = user_agent + " " + headers["user-agent"] 1805 else: 1806 headers["user-agent"] = user_agent 1807 resp, content = request_orig( 1808 uri, 1809 method=method, 1810 body=body, 1811 headers=headers, 1812 redirections=redirections, 1813 connection_type=connection_type, 1814 ) 1815 return resp, content 1816 1817 http.request = new_request 1818 return http 1819 1820 1821def tunnel_patch(http): 1822 """Tunnel PATCH requests over POST. 1823 Args: 1824 http - An instance of httplib2.Http 1825 or something that acts like it. 1826 1827 Returns: 1828 A modified instance of http that was passed in. 1829 1830 Example: 1831 1832 h = httplib2.Http() 1833 h = tunnel_patch(h, "my-app-name/6.0") 1834 1835 Useful if you are running on a platform that doesn't support PATCH. 1836 Apply this last if you are using OAuth 1.0, as changing the method 1837 will result in a different signature. 1838 """ 1839 request_orig = http.request 1840 1841 # The closure that will replace 'httplib2.Http.request'. 1842 def new_request( 1843 uri, 1844 method="GET", 1845 body=None, 1846 headers=None, 1847 redirections=httplib2.DEFAULT_MAX_REDIRECTS, 1848 connection_type=None, 1849 ): 1850 """Modify the request headers to add the user-agent.""" 1851 if headers is None: 1852 headers = {} 1853 if method == "PATCH": 1854 if "oauth_token" in headers.get("authorization", ""): 1855 LOGGER.warning( 1856 "OAuth 1.0 request made with Credentials after tunnel_patch." 1857 ) 1858 headers["x-http-method-override"] = "PATCH" 1859 method = "POST" 1860 resp, content = request_orig( 1861 uri, 1862 method=method, 1863 body=body, 1864 headers=headers, 1865 redirections=redirections, 1866 connection_type=connection_type, 1867 ) 1868 return resp, content 1869 1870 http.request = new_request 1871 return http 1872 1873 1874def build_http(): 1875 """Builds httplib2.Http object 1876 1877 Returns: 1878 A httplib2.Http object, which is used to make http requests, and which has timeout set by default. 1879 To override default timeout call 1880 1881 socket.setdefaulttimeout(timeout_in_sec) 1882 1883 before interacting with this method. 1884 """ 1885 if socket.getdefaulttimeout() is not None: 1886 http_timeout = socket.getdefaulttimeout() 1887 else: 1888 http_timeout = DEFAULT_HTTP_TIMEOUT_SEC 1889 http = httplib2.Http(timeout=http_timeout) 1890 # 308's are used by several Google APIs (Drive, YouTube) 1891 # for Resumable Uploads rather than Permanent Redirects. 1892 # This asks httplib2 to exclude 308s from the status codes 1893 # it treats as redirects 1894 http.redirect_codes = http.redirect_codes - {308} 1895 1896 return http 1897