1import unittest 2from test import support 3from test.support import socket_helper 4from test import test_urllib 5 6import os 7import io 8import socket 9import array 10import sys 11import tempfile 12import subprocess 13 14import urllib.request 15# The proxy bypass method imported below has logic specific to the OSX 16# proxy config data structure but is testable on all platforms. 17from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, 18 HTTPPasswordMgrWithPriorAuth, _parse_proxy, 19 _proxy_bypass_macosx_sysconf, 20 AbstractDigestAuthHandler) 21from urllib.parse import urlparse 22import urllib.error 23import http.client 24 25# XXX 26# Request 27# CacheFTPHandler (hard to write) 28# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler 29 30 31class TrivialTests(unittest.TestCase): 32 33 def test___all__(self): 34 # Verify which names are exposed 35 for module in 'request', 'response', 'parse', 'error', 'robotparser': 36 context = {} 37 exec('from urllib.%s import *' % module, context) 38 del context['__builtins__'] 39 if module == 'request' and os.name == 'nt': 40 u, p = context.pop('url2pathname'), context.pop('pathname2url') 41 self.assertEqual(u.__module__, 'nturl2path') 42 self.assertEqual(p.__module__, 'nturl2path') 43 for k, v in context.items(): 44 self.assertEqual(v.__module__, 'urllib.%s' % module, 45 "%r is exposed in 'urllib.%s' but defined in %r" % 46 (k, module, v.__module__)) 47 48 def test_trivial(self): 49 # A couple trivial tests 50 51 # clear _opener global variable 52 self.addCleanup(urllib.request.urlcleanup) 53 54 self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') 55 56 # XXX Name hacking to get this to work on Windows. 57 fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') 58 59 if os.name == 'nt': 60 file_url = "file:///%s" % fname 61 else: 62 file_url = "file://%s" % fname 63 64 with urllib.request.urlopen(file_url) as f: 65 f.read() 66 67 def test_parse_http_list(self): 68 tests = [ 69 ('a,b,c', ['a', 'b', 'c']), 70 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), 71 ('a, b, "c", "d", "e,f", g, h', 72 ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), 73 ('a="b\\"c", d="e\\,f", g="h\\\\i"', 74 ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] 75 for string, list in tests: 76 self.assertEqual(urllib.request.parse_http_list(string), list) 77 78 def test_URLError_reasonstr(self): 79 err = urllib.error.URLError('reason') 80 self.assertIn(err.reason, str(err)) 81 82 83class RequestHdrsTests(unittest.TestCase): 84 85 def test_request_headers_dict(self): 86 """ 87 The Request.headers dictionary is not a documented interface. It 88 should stay that way, because the complete set of headers are only 89 accessible through the .get_header(), .has_header(), .header_items() 90 interface. However, .headers pre-dates those methods, and so real code 91 will be using the dictionary. 92 93 The introduction in 2.4 of those methods was a mistake for the same 94 reason: code that previously saw all (urllib2 user)-provided headers in 95 .headers now sees only a subset. 96 97 """ 98 url = "http://example.com" 99 self.assertEqual(Request(url, 100 headers={"Spam-eggs": "blah"} 101 ).headers["Spam-eggs"], "blah") 102 self.assertEqual(Request(url, 103 headers={"spam-EggS": "blah"} 104 ).headers["Spam-eggs"], "blah") 105 106 def test_request_headers_methods(self): 107 """ 108 Note the case normalization of header names here, to 109 .capitalize()-case. This should be preserved for 110 backwards-compatibility. (In the HTTP case, normalization to 111 .title()-case is done by urllib2 before sending headers to 112 http.client). 113 114 Note that e.g. r.has_header("spam-EggS") is currently False, and 115 r.get_header("spam-EggS") returns None, but that could be changed in 116 future. 117 118 Method r.remove_header should remove items both from r.headers and 119 r.unredirected_hdrs dictionaries 120 """ 121 url = "http://example.com" 122 req = Request(url, headers={"Spam-eggs": "blah"}) 123 self.assertTrue(req.has_header("Spam-eggs")) 124 self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) 125 126 req.add_header("Foo-Bar", "baz") 127 self.assertEqual(sorted(req.header_items()), 128 [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) 129 self.assertFalse(req.has_header("Not-there")) 130 self.assertIsNone(req.get_header("Not-there")) 131 self.assertEqual(req.get_header("Not-there", "default"), "default") 132 133 req.remove_header("Spam-eggs") 134 self.assertFalse(req.has_header("Spam-eggs")) 135 136 req.add_unredirected_header("Unredirected-spam", "Eggs") 137 self.assertTrue(req.has_header("Unredirected-spam")) 138 139 req.remove_header("Unredirected-spam") 140 self.assertFalse(req.has_header("Unredirected-spam")) 141 142 def test_password_manager(self): 143 mgr = urllib.request.HTTPPasswordMgr() 144 add = mgr.add_password 145 find_user_pass = mgr.find_user_password 146 147 add("Some Realm", "http://example.com/", "joe", "password") 148 add("Some Realm", "http://example.com/ni", "ni", "ni") 149 add("Some Realm", "http://c.example.com:3128", "3", "c") 150 add("Some Realm", "d.example.com", "4", "d") 151 add("Some Realm", "e.example.com:3128", "5", "e") 152 153 # For the same realm, password set the highest path is the winner. 154 self.assertEqual(find_user_pass("Some Realm", "example.com"), 155 ('joe', 'password')) 156 self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), 157 ('joe', 'password')) 158 self.assertEqual(find_user_pass("Some Realm", "http://example.com"), 159 ('joe', 'password')) 160 self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), 161 ('joe', 'password')) 162 self.assertEqual(find_user_pass("Some Realm", 163 "http://example.com/spam"), 164 ('joe', 'password')) 165 166 self.assertEqual(find_user_pass("Some Realm", 167 "http://example.com/spam/spam"), 168 ('joe', 'password')) 169 170 # You can have different passwords for different paths. 171 172 add("c", "http://example.com/foo", "foo", "ni") 173 add("c", "http://example.com/bar", "bar", "nini") 174 175 self.assertEqual(find_user_pass("c", "http://example.com/foo"), 176 ('foo', 'ni')) 177 178 self.assertEqual(find_user_pass("c", "http://example.com/bar"), 179 ('bar', 'nini')) 180 181 # For the same path, newer password should be considered. 182 183 add("b", "http://example.com/", "first", "blah") 184 add("b", "http://example.com/", "second", "spam") 185 186 self.assertEqual(find_user_pass("b", "http://example.com/"), 187 ('second', 'spam')) 188 189 # No special relationship between a.example.com and example.com: 190 191 add("a", "http://example.com", "1", "a") 192 self.assertEqual(find_user_pass("a", "http://example.com/"), 193 ('1', 'a')) 194 195 self.assertEqual(find_user_pass("a", "http://a.example.com/"), 196 (None, None)) 197 198 # Ports: 199 200 self.assertEqual(find_user_pass("Some Realm", "c.example.com"), 201 (None, None)) 202 self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), 203 ('3', 'c')) 204 self.assertEqual( 205 find_user_pass("Some Realm", "http://c.example.com:3128"), 206 ('3', 'c')) 207 self.assertEqual(find_user_pass("Some Realm", "d.example.com"), 208 ('4', 'd')) 209 self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), 210 ('5', 'e')) 211 212 def test_password_manager_default_port(self): 213 """ 214 The point to note here is that we can't guess the default port if 215 there's no scheme. This applies to both add_password and 216 find_user_password. 217 """ 218 mgr = urllib.request.HTTPPasswordMgr() 219 add = mgr.add_password 220 find_user_pass = mgr.find_user_password 221 add("f", "http://g.example.com:80", "10", "j") 222 add("g", "http://h.example.com", "11", "k") 223 add("h", "i.example.com:80", "12", "l") 224 add("i", "j.example.com", "13", "m") 225 self.assertEqual(find_user_pass("f", "g.example.com:100"), 226 (None, None)) 227 self.assertEqual(find_user_pass("f", "g.example.com:80"), 228 ('10', 'j')) 229 self.assertEqual(find_user_pass("f", "g.example.com"), 230 (None, None)) 231 self.assertEqual(find_user_pass("f", "http://g.example.com:100"), 232 (None, None)) 233 self.assertEqual(find_user_pass("f", "http://g.example.com:80"), 234 ('10', 'j')) 235 self.assertEqual(find_user_pass("f", "http://g.example.com"), 236 ('10', 'j')) 237 self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) 238 self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) 239 self.assertEqual(find_user_pass("g", "http://h.example.com:80"), 240 ('11', 'k')) 241 self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) 242 self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) 243 self.assertEqual(find_user_pass("h", "http://i.example.com:80"), 244 ('12', 'l')) 245 self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) 246 self.assertEqual(find_user_pass("i", "j.example.com:80"), 247 (None, None)) 248 self.assertEqual(find_user_pass("i", "http://j.example.com"), 249 ('13', 'm')) 250 self.assertEqual(find_user_pass("i", "http://j.example.com:80"), 251 (None, None)) 252 253 254class MockOpener: 255 addheaders = [] 256 257 def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 258 self.req, self.data, self.timeout = req, data, timeout 259 260 def error(self, proto, *args): 261 self.proto, self.args = proto, args 262 263 264class MockFile: 265 def read(self, count=None): 266 pass 267 268 def readline(self, count=None): 269 pass 270 271 def close(self): 272 pass 273 274 275class MockHeaders(dict): 276 def getheaders(self, name): 277 return list(self.values()) 278 279 280class MockResponse(io.StringIO): 281 def __init__(self, code, msg, headers, data, url=None): 282 io.StringIO.__init__(self, data) 283 self.code, self.msg, self.headers, self.url = code, msg, headers, url 284 285 def info(self): 286 return self.headers 287 288 def geturl(self): 289 return self.url 290 291 292class MockCookieJar: 293 def add_cookie_header(self, request): 294 self.ach_req = request 295 296 def extract_cookies(self, response, request): 297 self.ec_req, self.ec_r = request, response 298 299 300class FakeMethod: 301 def __init__(self, meth_name, action, handle): 302 self.meth_name = meth_name 303 self.handle = handle 304 self.action = action 305 306 def __call__(self, *args): 307 return self.handle(self.meth_name, self.action, *args) 308 309 310class MockHTTPResponse(io.IOBase): 311 def __init__(self, fp, msg, status, reason): 312 self.fp = fp 313 self.msg = msg 314 self.status = status 315 self.reason = reason 316 self.code = 200 317 318 def read(self): 319 return '' 320 321 def info(self): 322 return {} 323 324 def geturl(self): 325 return self.url 326 327 328class MockHTTPClass: 329 def __init__(self): 330 self.level = 0 331 self.req_headers = [] 332 self.data = None 333 self.raise_on_endheaders = False 334 self.sock = None 335 self._tunnel_headers = {} 336 337 def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 338 self.host = host 339 self.timeout = timeout 340 return self 341 342 def set_debuglevel(self, level): 343 self.level = level 344 345 def set_tunnel(self, host, port=None, headers=None): 346 self._tunnel_host = host 347 self._tunnel_port = port 348 if headers: 349 self._tunnel_headers = headers 350 else: 351 self._tunnel_headers.clear() 352 353 def request(self, method, url, body=None, headers=None, *, 354 encode_chunked=False): 355 self.method = method 356 self.selector = url 357 if headers is not None: 358 self.req_headers += headers.items() 359 self.req_headers.sort() 360 if body: 361 self.data = body 362 self.encode_chunked = encode_chunked 363 if self.raise_on_endheaders: 364 raise OSError() 365 366 def getresponse(self): 367 return MockHTTPResponse(MockFile(), {}, 200, "OK") 368 369 def close(self): 370 pass 371 372 373class MockHandler: 374 # useful for testing handler machinery 375 # see add_ordered_mock_handlers() docstring 376 handler_order = 500 377 378 def __init__(self, methods): 379 self._define_methods(methods) 380 381 def _define_methods(self, methods): 382 for spec in methods: 383 if len(spec) == 2: 384 name, action = spec 385 else: 386 name, action = spec, None 387 meth = FakeMethod(name, action, self.handle) 388 setattr(self.__class__, name, meth) 389 390 def handle(self, fn_name, action, *args, **kwds): 391 self.parent.calls.append((self, fn_name, args, kwds)) 392 if action is None: 393 return None 394 elif action == "return self": 395 return self 396 elif action == "return response": 397 res = MockResponse(200, "OK", {}, "") 398 return res 399 elif action == "return request": 400 return Request("http://blah/") 401 elif action.startswith("error"): 402 code = action[action.rfind(" ")+1:] 403 try: 404 code = int(code) 405 except ValueError: 406 pass 407 res = MockResponse(200, "OK", {}, "") 408 return self.parent.error("http", args[0], res, code, "", {}) 409 elif action == "raise": 410 raise urllib.error.URLError("blah") 411 assert False 412 413 def close(self): 414 pass 415 416 def add_parent(self, parent): 417 self.parent = parent 418 self.parent.calls = [] 419 420 def __lt__(self, other): 421 if not hasattr(other, "handler_order"): 422 # No handler_order, leave in original order. Yuck. 423 return True 424 return self.handler_order < other.handler_order 425 426 427def add_ordered_mock_handlers(opener, meth_spec): 428 """Create MockHandlers and add them to an OpenerDirector. 429 430 meth_spec: list of lists of tuples and strings defining methods to define 431 on handlers. eg: 432 433 [["http_error", "ftp_open"], ["http_open"]] 434 435 defines methods .http_error() and .ftp_open() on one handler, and 436 .http_open() on another. These methods just record their arguments and 437 return None. Using a tuple instead of a string causes the method to 438 perform some action (see MockHandler.handle()), eg: 439 440 [["http_error"], [("http_open", "return request")]] 441 442 defines .http_error() on one handler (which simply returns None), and 443 .http_open() on another handler, which returns a Request object. 444 445 """ 446 handlers = [] 447 count = 0 448 for meths in meth_spec: 449 class MockHandlerSubclass(MockHandler): 450 pass 451 452 h = MockHandlerSubclass(meths) 453 h.handler_order += count 454 h.add_parent(opener) 455 count = count + 1 456 handlers.append(h) 457 opener.add_handler(h) 458 return handlers 459 460 461def build_test_opener(*handler_instances): 462 opener = OpenerDirector() 463 for h in handler_instances: 464 opener.add_handler(h) 465 return opener 466 467 468class MockHTTPHandler(urllib.request.BaseHandler): 469 # useful for testing redirections and auth 470 # sends supplied headers and code as first response 471 # sends 200 OK as second response 472 def __init__(self, code, headers): 473 self.code = code 474 self.headers = headers 475 self.reset() 476 477 def reset(self): 478 self._count = 0 479 self.requests = [] 480 481 def http_open(self, req): 482 import email, copy 483 self.requests.append(copy.deepcopy(req)) 484 if self._count == 0: 485 self._count = self._count + 1 486 name = http.client.responses[self.code] 487 msg = email.message_from_string(self.headers) 488 return self.parent.error( 489 "http", req, MockFile(), self.code, name, msg) 490 else: 491 self.req = req 492 msg = email.message_from_string("\r\n\r\n") 493 return MockResponse(200, "OK", msg, "", req.get_full_url()) 494 495 496class MockHTTPSHandler(urllib.request.AbstractHTTPHandler): 497 # Useful for testing the Proxy-Authorization request by verifying the 498 # properties of httpcon 499 500 def __init__(self, debuglevel=0): 501 urllib.request.AbstractHTTPHandler.__init__(self, debuglevel=debuglevel) 502 self.httpconn = MockHTTPClass() 503 504 def https_open(self, req): 505 return self.do_open(self.httpconn, req) 506 507 508class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): 509 # useful for testing auth 510 # sends supplied code response 511 # checks if auth header is specified in request 512 def __init__(self, code): 513 self.code = code 514 self.has_auth_header = False 515 516 def reset(self): 517 self.has_auth_header = False 518 519 def http_open(self, req): 520 if req.has_header('Authorization'): 521 self.has_auth_header = True 522 name = http.client.responses[self.code] 523 return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) 524 525 526 527class MockPasswordManager: 528 def add_password(self, realm, uri, user, password): 529 self.realm = realm 530 self.url = uri 531 self.user = user 532 self.password = password 533 534 def find_user_password(self, realm, authuri): 535 self.target_realm = realm 536 self.target_url = authuri 537 return self.user, self.password 538 539 540class OpenerDirectorTests(unittest.TestCase): 541 542 def test_add_non_handler(self): 543 class NonHandler(object): 544 pass 545 self.assertRaises(TypeError, 546 OpenerDirector().add_handler, NonHandler()) 547 548 def test_badly_named_methods(self): 549 # test work-around for three methods that accidentally follow the 550 # naming conventions for handler methods 551 # (*_open() / *_request() / *_response()) 552 553 # These used to call the accidentally-named methods, causing a 554 # TypeError in real code; here, returning self from these mock 555 # methods would either cause no exception, or AttributeError. 556 557 from urllib.error import URLError 558 559 o = OpenerDirector() 560 meth_spec = [ 561 [("do_open", "return self"), ("proxy_open", "return self")], 562 [("redirect_request", "return self")], 563 ] 564 add_ordered_mock_handlers(o, meth_spec) 565 o.add_handler(urllib.request.UnknownHandler()) 566 for scheme in "do", "proxy", "redirect": 567 self.assertRaises(URLError, o.open, scheme+"://example.com/") 568 569 def test_handled(self): 570 # handler returning non-None means no more handlers will be called 571 o = OpenerDirector() 572 meth_spec = [ 573 ["http_open", "ftp_open", "http_error_302"], 574 ["ftp_open"], 575 [("http_open", "return self")], 576 [("http_open", "return self")], 577 ] 578 handlers = add_ordered_mock_handlers(o, meth_spec) 579 580 req = Request("http://example.com/") 581 r = o.open(req) 582 # Second .http_open() gets called, third doesn't, since second returned 583 # non-None. Handlers without .http_open() never get any methods called 584 # on them. 585 # In fact, second mock handler defining .http_open() returns self 586 # (instead of response), which becomes the OpenerDirector's return 587 # value. 588 self.assertEqual(r, handlers[2]) 589 calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] 590 for expected, got in zip(calls, o.calls): 591 handler, name, args, kwds = got 592 self.assertEqual((handler, name), expected) 593 self.assertEqual(args, (req,)) 594 595 def test_handler_order(self): 596 o = OpenerDirector() 597 handlers = [] 598 for meths, handler_order in [([("http_open", "return self")], 500), 599 (["http_open"], 0)]: 600 class MockHandlerSubclass(MockHandler): 601 pass 602 603 h = MockHandlerSubclass(meths) 604 h.handler_order = handler_order 605 handlers.append(h) 606 o.add_handler(h) 607 608 o.open("http://example.com/") 609 # handlers called in reverse order, thanks to their sort order 610 self.assertEqual(o.calls[0][0], handlers[1]) 611 self.assertEqual(o.calls[1][0], handlers[0]) 612 613 def test_raise(self): 614 # raising URLError stops processing of request 615 o = OpenerDirector() 616 meth_spec = [ 617 [("http_open", "raise")], 618 [("http_open", "return self")], 619 ] 620 handlers = add_ordered_mock_handlers(o, meth_spec) 621 622 req = Request("http://example.com/") 623 self.assertRaises(urllib.error.URLError, o.open, req) 624 self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) 625 626 def test_http_error(self): 627 # XXX http_error_default 628 # http errors are a special case 629 o = OpenerDirector() 630 meth_spec = [ 631 [("http_open", "error 302")], 632 [("http_error_400", "raise"), "http_open"], 633 [("http_error_302", "return response"), "http_error_303", 634 "http_error"], 635 [("http_error_302")], 636 ] 637 handlers = add_ordered_mock_handlers(o, meth_spec) 638 req = Request("http://example.com/") 639 o.open(req) 640 assert len(o.calls) == 2 641 calls = [(handlers[0], "http_open", (req,)), 642 (handlers[2], "http_error_302", 643 (req, support.ALWAYS_EQ, 302, "", {}))] 644 for expected, got in zip(calls, o.calls): 645 handler, method_name, args = expected 646 self.assertEqual((handler, method_name), got[:2]) 647 self.assertEqual(args, got[2]) 648 649 def test_processors(self): 650 # *_request / *_response methods get called appropriately 651 o = OpenerDirector() 652 meth_spec = [ 653 [("http_request", "return request"), 654 ("http_response", "return response")], 655 [("http_request", "return request"), 656 ("http_response", "return response")], 657 ] 658 handlers = add_ordered_mock_handlers(o, meth_spec) 659 660 req = Request("http://example.com/") 661 o.open(req) 662 # processor methods are called on *all* handlers that define them, 663 # not just the first handler that handles the request 664 calls = [ 665 (handlers[0], "http_request"), (handlers[1], "http_request"), 666 (handlers[0], "http_response"), (handlers[1], "http_response")] 667 668 for i, (handler, name, args, kwds) in enumerate(o.calls): 669 if i < 2: 670 # *_request 671 self.assertEqual((handler, name), calls[i]) 672 self.assertEqual(len(args), 1) 673 self.assertIsInstance(args[0], Request) 674 else: 675 # *_response 676 self.assertEqual((handler, name), calls[i]) 677 self.assertEqual(len(args), 2) 678 self.assertIsInstance(args[0], Request) 679 # response from opener.open is None, because there's no 680 # handler that defines http_open to handle it 681 if args[1] is not None: 682 self.assertIsInstance(args[1], MockResponse) 683 684 685def sanepathname2url(path): 686 try: 687 path.encode("utf-8") 688 except UnicodeEncodeError: 689 raise unittest.SkipTest("path is not encodable to utf8") 690 urlpath = urllib.request.pathname2url(path) 691 if os.name == "nt" and urlpath.startswith("///"): 692 urlpath = urlpath[2:] 693 # XXX don't ask me about the mac... 694 return urlpath 695 696 697class HandlerTests(unittest.TestCase): 698 699 def test_ftp(self): 700 class MockFTPWrapper: 701 def __init__(self, data): 702 self.data = data 703 704 def retrfile(self, filename, filetype): 705 self.filename, self.filetype = filename, filetype 706 return io.StringIO(self.data), len(self.data) 707 708 def close(self): 709 pass 710 711 class NullFTPHandler(urllib.request.FTPHandler): 712 def __init__(self, data): 713 self.data = data 714 715 def connect_ftp(self, user, passwd, host, port, dirs, 716 timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 717 self.user, self.passwd = user, passwd 718 self.host, self.port = host, port 719 self.dirs = dirs 720 self.ftpwrapper = MockFTPWrapper(self.data) 721 return self.ftpwrapper 722 723 import ftplib 724 data = "rheum rhaponicum" 725 h = NullFTPHandler(data) 726 h.parent = MockOpener() 727 728 for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ 729 ("ftp://localhost/foo/bar/baz.html", 730 "localhost", ftplib.FTP_PORT, "", "", "I", 731 ["foo", "bar"], "baz.html", "text/html"), 732 ("ftp://parrot@localhost/foo/bar/baz.html", 733 "localhost", ftplib.FTP_PORT, "parrot", "", "I", 734 ["foo", "bar"], "baz.html", "text/html"), 735 ("ftp://%25parrot@localhost/foo/bar/baz.html", 736 "localhost", ftplib.FTP_PORT, "%parrot", "", "I", 737 ["foo", "bar"], "baz.html", "text/html"), 738 ("ftp://%2542parrot@localhost/foo/bar/baz.html", 739 "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", 740 ["foo", "bar"], "baz.html", "text/html"), 741 ("ftp://localhost:80/foo/bar/", 742 "localhost", 80, "", "", "D", 743 ["foo", "bar"], "", None), 744 ("ftp://localhost/baz.gif;type=a", 745 "localhost", ftplib.FTP_PORT, "", "", "A", 746 [], "baz.gif", None), # XXX really this should guess image/gif 747 ]: 748 req = Request(url) 749 req.timeout = None 750 r = h.ftp_open(req) 751 # ftp authentication not yet implemented by FTPHandler 752 self.assertEqual(h.user, user) 753 self.assertEqual(h.passwd, passwd) 754 self.assertEqual(h.host, socket.gethostbyname(host)) 755 self.assertEqual(h.port, port) 756 self.assertEqual(h.dirs, dirs) 757 self.assertEqual(h.ftpwrapper.filename, filename) 758 self.assertEqual(h.ftpwrapper.filetype, type_) 759 headers = r.info() 760 self.assertEqual(headers.get("Content-type"), mimetype) 761 self.assertEqual(int(headers["Content-length"]), len(data)) 762 763 def test_file(self): 764 import email.utils 765 h = urllib.request.FileHandler() 766 o = h.parent = MockOpener() 767 768 TESTFN = support.TESTFN 769 urlpath = sanepathname2url(os.path.abspath(TESTFN)) 770 towrite = b"hello, world\n" 771 urls = [ 772 "file://localhost%s" % urlpath, 773 "file://%s" % urlpath, 774 "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), 775 ] 776 try: 777 localaddr = socket.gethostbyname(socket.gethostname()) 778 except socket.gaierror: 779 localaddr = '' 780 if localaddr: 781 urls.append("file://%s%s" % (localaddr, urlpath)) 782 783 for url in urls: 784 f = open(TESTFN, "wb") 785 try: 786 try: 787 f.write(towrite) 788 finally: 789 f.close() 790 791 r = h.file_open(Request(url)) 792 try: 793 data = r.read() 794 headers = r.info() 795 respurl = r.geturl() 796 finally: 797 r.close() 798 stats = os.stat(TESTFN) 799 modified = email.utils.formatdate(stats.st_mtime, usegmt=True) 800 finally: 801 os.remove(TESTFN) 802 self.assertEqual(data, towrite) 803 self.assertEqual(headers["Content-type"], "text/plain") 804 self.assertEqual(headers["Content-length"], "13") 805 self.assertEqual(headers["Last-modified"], modified) 806 self.assertEqual(respurl, url) 807 808 for url in [ 809 "file://localhost:80%s" % urlpath, 810 "file:///file_does_not_exist.txt", 811 "file://not-a-local-host.com//dir/file.txt", 812 "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), 813 os.getcwd(), TESTFN), 814 "file://somerandomhost.ontheinternet.com%s/%s" % 815 (os.getcwd(), TESTFN), 816 ]: 817 try: 818 f = open(TESTFN, "wb") 819 try: 820 f.write(towrite) 821 finally: 822 f.close() 823 824 self.assertRaises(urllib.error.URLError, 825 h.file_open, Request(url)) 826 finally: 827 os.remove(TESTFN) 828 829 h = urllib.request.FileHandler() 830 o = h.parent = MockOpener() 831 # XXXX why does // mean ftp (and /// mean not ftp!), and where 832 # is file: scheme specified? I think this is really a bug, and 833 # what was intended was to distinguish between URLs like: 834 # file:/blah.txt (a file) 835 # file://localhost/blah.txt (a file) 836 # file:///blah.txt (a file) 837 # file://ftp.example.com/blah.txt (an ftp URL) 838 for url, ftp in [ 839 ("file://ftp.example.com//foo.txt", False), 840 ("file://ftp.example.com///foo.txt", False), 841 ("file://ftp.example.com/foo.txt", False), 842 ("file://somehost//foo/something.txt", False), 843 ("file://localhost//foo/something.txt", False), 844 ]: 845 req = Request(url) 846 try: 847 h.file_open(req) 848 except urllib.error.URLError: 849 self.assertFalse(ftp) 850 else: 851 self.assertIs(o.req, req) 852 self.assertEqual(req.type, "ftp") 853 self.assertEqual(req.type == "ftp", ftp) 854 855 def test_http(self): 856 857 h = urllib.request.AbstractHTTPHandler() 858 o = h.parent = MockOpener() 859 860 url = "http://example.com/" 861 for method, data in [("GET", None), ("POST", b"blah")]: 862 req = Request(url, data, {"Foo": "bar"}) 863 req.timeout = None 864 req.add_unredirected_header("Spam", "eggs") 865 http = MockHTTPClass() 866 r = h.do_open(http, req) 867 868 # result attributes 869 r.read; r.readline # wrapped MockFile methods 870 r.info; r.geturl # addinfourl methods 871 r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() 872 hdrs = r.info() 873 hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() 874 self.assertEqual(r.geturl(), url) 875 876 self.assertEqual(http.host, "example.com") 877 self.assertEqual(http.level, 0) 878 self.assertEqual(http.method, method) 879 self.assertEqual(http.selector, "/") 880 self.assertEqual(http.req_headers, 881 [("Connection", "close"), 882 ("Foo", "bar"), ("Spam", "eggs")]) 883 self.assertEqual(http.data, data) 884 885 # check OSError converted to URLError 886 http.raise_on_endheaders = True 887 self.assertRaises(urllib.error.URLError, h.do_open, http, req) 888 889 # Check for TypeError on POST data which is str. 890 req = Request("http://example.com/","badpost") 891 self.assertRaises(TypeError, h.do_request_, req) 892 893 # check adding of standard headers 894 o.addheaders = [("Spam", "eggs")] 895 for data in b"", None: # POST, GET 896 req = Request("http://example.com/", data) 897 r = MockResponse(200, "OK", {}, "") 898 newreq = h.do_request_(req) 899 if data is None: # GET 900 self.assertNotIn("Content-length", req.unredirected_hdrs) 901 self.assertNotIn("Content-type", req.unredirected_hdrs) 902 else: # POST 903 self.assertEqual(req.unredirected_hdrs["Content-length"], "0") 904 self.assertEqual(req.unredirected_hdrs["Content-type"], 905 "application/x-www-form-urlencoded") 906 # XXX the details of Host could be better tested 907 self.assertEqual(req.unredirected_hdrs["Host"], "example.com") 908 self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") 909 910 # don't clobber existing headers 911 req.add_unredirected_header("Content-length", "foo") 912 req.add_unredirected_header("Content-type", "bar") 913 req.add_unredirected_header("Host", "baz") 914 req.add_unredirected_header("Spam", "foo") 915 newreq = h.do_request_(req) 916 self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") 917 self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") 918 self.assertEqual(req.unredirected_hdrs["Host"], "baz") 919 self.assertEqual(req.unredirected_hdrs["Spam"], "foo") 920 921 def test_http_body_file(self): 922 # A regular file - chunked encoding is used unless Content Length is 923 # already set. 924 925 h = urllib.request.AbstractHTTPHandler() 926 o = h.parent = MockOpener() 927 928 file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) 929 file_path = file_obj.name 930 file_obj.close() 931 self.addCleanup(os.unlink, file_path) 932 933 with open(file_path, "rb") as f: 934 req = Request("http://example.com/", f, {}) 935 newreq = h.do_request_(req) 936 te = newreq.get_header('Transfer-encoding') 937 self.assertEqual(te, "chunked") 938 self.assertFalse(newreq.has_header('Content-length')) 939 940 with open(file_path, "rb") as f: 941 req = Request("http://example.com/", f, {"Content-Length": 30}) 942 newreq = h.do_request_(req) 943 self.assertEqual(int(newreq.get_header('Content-length')), 30) 944 self.assertFalse(newreq.has_header("Transfer-encoding")) 945 946 def test_http_body_fileobj(self): 947 # A file object - chunked encoding is used 948 # unless Content Length is already set. 949 # (Note that there are some subtle differences to a regular 950 # file, that is why we are testing both cases.) 951 952 h = urllib.request.AbstractHTTPHandler() 953 o = h.parent = MockOpener() 954 file_obj = io.BytesIO() 955 956 req = Request("http://example.com/", file_obj, {}) 957 newreq = h.do_request_(req) 958 self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') 959 self.assertFalse(newreq.has_header('Content-length')) 960 961 headers = {"Content-Length": 30} 962 req = Request("http://example.com/", file_obj, headers) 963 newreq = h.do_request_(req) 964 self.assertEqual(int(newreq.get_header('Content-length')), 30) 965 self.assertFalse(newreq.has_header("Transfer-encoding")) 966 967 file_obj.close() 968 969 def test_http_body_pipe(self): 970 # A file reading from a pipe. 971 # A pipe cannot be seek'ed. There is no way to determine the 972 # content length up front. Thus, do_request_() should fall 973 # back to Transfer-encoding chunked. 974 975 h = urllib.request.AbstractHTTPHandler() 976 o = h.parent = MockOpener() 977 978 cmd = [sys.executable, "-c", r"pass"] 979 for headers in {}, {"Content-Length": 30}: 980 with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: 981 req = Request("http://example.com/", proc.stdout, headers) 982 newreq = h.do_request_(req) 983 if not headers: 984 self.assertEqual(newreq.get_header('Content-length'), None) 985 self.assertEqual(newreq.get_header('Transfer-encoding'), 986 'chunked') 987 else: 988 self.assertEqual(int(newreq.get_header('Content-length')), 989 30) 990 991 def test_http_body_iterable(self): 992 # Generic iterable. There is no way to determine the content 993 # length up front. Fall back to Transfer-encoding chunked. 994 995 h = urllib.request.AbstractHTTPHandler() 996 o = h.parent = MockOpener() 997 998 def iterable_body(): 999 yield b"one" 1000 1001 for headers in {}, {"Content-Length": 11}: 1002 req = Request("http://example.com/", iterable_body(), headers) 1003 newreq = h.do_request_(req) 1004 if not headers: 1005 self.assertEqual(newreq.get_header('Content-length'), None) 1006 self.assertEqual(newreq.get_header('Transfer-encoding'), 1007 'chunked') 1008 else: 1009 self.assertEqual(int(newreq.get_header('Content-length')), 11) 1010 1011 def test_http_body_empty_seq(self): 1012 # Zero-length iterable body should be treated like any other iterable 1013 h = urllib.request.AbstractHTTPHandler() 1014 h.parent = MockOpener() 1015 req = h.do_request_(Request("http://example.com/", ())) 1016 self.assertEqual(req.get_header("Transfer-encoding"), "chunked") 1017 self.assertFalse(req.has_header("Content-length")) 1018 1019 def test_http_body_array(self): 1020 # array.array Iterable - Content Length is calculated 1021 1022 h = urllib.request.AbstractHTTPHandler() 1023 o = h.parent = MockOpener() 1024 1025 iterable_array = array.array("I",[1,2,3,4]) 1026 1027 for headers in {}, {"Content-Length": 16}: 1028 req = Request("http://example.com/", iterable_array, headers) 1029 newreq = h.do_request_(req) 1030 self.assertEqual(int(newreq.get_header('Content-length')),16) 1031 1032 def test_http_handler_debuglevel(self): 1033 o = OpenerDirector() 1034 h = MockHTTPSHandler(debuglevel=1) 1035 o.add_handler(h) 1036 o.open("https://www.example.com") 1037 self.assertEqual(h._debuglevel, 1) 1038 1039 def test_http_doubleslash(self): 1040 # Checks the presence of any unnecessary double slash in url does not 1041 # break anything. Previously, a double slash directly after the host 1042 # could cause incorrect parsing. 1043 h = urllib.request.AbstractHTTPHandler() 1044 h.parent = MockOpener() 1045 1046 data = b"" 1047 ds_urls = [ 1048 "http://example.com/foo/bar/baz.html", 1049 "http://example.com//foo/bar/baz.html", 1050 "http://example.com/foo//bar/baz.html", 1051 "http://example.com/foo/bar//baz.html" 1052 ] 1053 1054 for ds_url in ds_urls: 1055 ds_req = Request(ds_url, data) 1056 1057 # Check whether host is determined correctly if there is no proxy 1058 np_ds_req = h.do_request_(ds_req) 1059 self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") 1060 1061 # Check whether host is determined correctly if there is a proxy 1062 ds_req.set_proxy("someproxy:3128", None) 1063 p_ds_req = h.do_request_(ds_req) 1064 self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") 1065 1066 def test_full_url_setter(self): 1067 # Checks to ensure that components are set correctly after setting the 1068 # full_url of a Request object 1069 1070 urls = [ 1071 'http://example.com?foo=bar#baz', 1072 'http://example.com?foo=bar&spam=eggs#bash', 1073 'http://example.com', 1074 ] 1075 1076 # testing a reusable request instance, but the url parameter is 1077 # required, so just use a dummy one to instantiate 1078 r = Request('http://example.com') 1079 for url in urls: 1080 r.full_url = url 1081 parsed = urlparse(url) 1082 1083 self.assertEqual(r.get_full_url(), url) 1084 # full_url setter uses splittag to split into components. 1085 # splittag sets the fragment as None while urlparse sets it to '' 1086 self.assertEqual(r.fragment or '', parsed.fragment) 1087 self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) 1088 1089 def test_full_url_deleter(self): 1090 r = Request('http://www.example.com') 1091 del r.full_url 1092 self.assertIsNone(r.full_url) 1093 self.assertIsNone(r.fragment) 1094 self.assertEqual(r.selector, '') 1095 1096 def test_fixpath_in_weirdurls(self): 1097 # Issue4493: urllib2 to supply '/' when to urls where path does not 1098 # start with'/' 1099 1100 h = urllib.request.AbstractHTTPHandler() 1101 h.parent = MockOpener() 1102 1103 weird_url = 'http://www.python.org?getspam' 1104 req = Request(weird_url) 1105 newreq = h.do_request_(req) 1106 self.assertEqual(newreq.host, 'www.python.org') 1107 self.assertEqual(newreq.selector, '/?getspam') 1108 1109 url_without_path = 'http://www.python.org' 1110 req = Request(url_without_path) 1111 newreq = h.do_request_(req) 1112 self.assertEqual(newreq.host, 'www.python.org') 1113 self.assertEqual(newreq.selector, '') 1114 1115 def test_errors(self): 1116 h = urllib.request.HTTPErrorProcessor() 1117 o = h.parent = MockOpener() 1118 1119 url = "http://example.com/" 1120 req = Request(url) 1121 # all 2xx are passed through 1122 r = MockResponse(200, "OK", {}, "", url) 1123 newr = h.http_response(req, r) 1124 self.assertIs(r, newr) 1125 self.assertFalse(hasattr(o, "proto")) # o.error not called 1126 r = MockResponse(202, "Accepted", {}, "", url) 1127 newr = h.http_response(req, r) 1128 self.assertIs(r, newr) 1129 self.assertFalse(hasattr(o, "proto")) # o.error not called 1130 r = MockResponse(206, "Partial content", {}, "", url) 1131 newr = h.http_response(req, r) 1132 self.assertIs(r, newr) 1133 self.assertFalse(hasattr(o, "proto")) # o.error not called 1134 # anything else calls o.error (and MockOpener returns None, here) 1135 r = MockResponse(502, "Bad gateway", {}, "", url) 1136 self.assertIsNone(h.http_response(req, r)) 1137 self.assertEqual(o.proto, "http") # o.error called 1138 self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) 1139 1140 def test_cookies(self): 1141 cj = MockCookieJar() 1142 h = urllib.request.HTTPCookieProcessor(cj) 1143 h.parent = MockOpener() 1144 1145 req = Request("http://example.com/") 1146 r = MockResponse(200, "OK", {}, "") 1147 newreq = h.http_request(req) 1148 self.assertIs(cj.ach_req, req) 1149 self.assertIs(cj.ach_req, newreq) 1150 self.assertEqual(req.origin_req_host, "example.com") 1151 self.assertFalse(req.unverifiable) 1152 newr = h.http_response(req, r) 1153 self.assertIs(cj.ec_req, req) 1154 self.assertIs(cj.ec_r, r) 1155 self.assertIs(r, newr) 1156 1157 def test_redirect(self): 1158 from_url = "http://example.com/a.html" 1159 to_url = "http://example.com/b.html" 1160 h = urllib.request.HTTPRedirectHandler() 1161 o = h.parent = MockOpener() 1162 1163 # ordinary redirect behaviour 1164 for code in 301, 302, 303, 307: 1165 for data in None, "blah\nblah\n": 1166 method = getattr(h, "http_error_%s" % code) 1167 req = Request(from_url, data) 1168 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1169 req.add_header("Nonsense", "viking=withhold") 1170 if data is not None: 1171 req.add_header("Content-Length", str(len(data))) 1172 req.add_unredirected_header("Spam", "spam") 1173 try: 1174 method(req, MockFile(), code, "Blah", 1175 MockHeaders({"location": to_url})) 1176 except urllib.error.HTTPError: 1177 # 307 in response to POST requires user OK 1178 self.assertEqual(code, 307) 1179 self.assertIsNotNone(data) 1180 self.assertEqual(o.req.get_full_url(), to_url) 1181 try: 1182 self.assertEqual(o.req.get_method(), "GET") 1183 except AttributeError: 1184 self.assertFalse(o.req.data) 1185 1186 # now it's a GET, there should not be headers regarding content 1187 # (possibly dragged from before being a POST) 1188 headers = [x.lower() for x in o.req.headers] 1189 self.assertNotIn("content-length", headers) 1190 self.assertNotIn("content-type", headers) 1191 1192 self.assertEqual(o.req.headers["Nonsense"], 1193 "viking=withhold") 1194 self.assertNotIn("Spam", o.req.headers) 1195 self.assertNotIn("Spam", o.req.unredirected_hdrs) 1196 1197 # loop detection 1198 req = Request(from_url) 1199 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1200 1201 def redirect(h, req, url=to_url): 1202 h.http_error_302(req, MockFile(), 302, "Blah", 1203 MockHeaders({"location": url})) 1204 # Note that the *original* request shares the same record of 1205 # redirections with the sub-requests caused by the redirections. 1206 1207 # detect infinite loop redirect of a URL to itself 1208 req = Request(from_url, origin_req_host="example.com") 1209 count = 0 1210 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1211 try: 1212 while 1: 1213 redirect(h, req, "http://example.com/") 1214 count = count + 1 1215 except urllib.error.HTTPError: 1216 # don't stop until max_repeats, because cookies may introduce state 1217 self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) 1218 1219 # detect endless non-repeating chain of redirects 1220 req = Request(from_url, origin_req_host="example.com") 1221 count = 0 1222 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1223 try: 1224 while 1: 1225 redirect(h, req, "http://example.com/%d" % count) 1226 count = count + 1 1227 except urllib.error.HTTPError: 1228 self.assertEqual(count, 1229 urllib.request.HTTPRedirectHandler.max_redirections) 1230 1231 def test_invalid_redirect(self): 1232 from_url = "http://example.com/a.html" 1233 valid_schemes = ['http','https','ftp'] 1234 invalid_schemes = ['file','imap','ldap'] 1235 schemeless_url = "example.com/b.html" 1236 h = urllib.request.HTTPRedirectHandler() 1237 o = h.parent = MockOpener() 1238 req = Request(from_url) 1239 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1240 1241 for scheme in invalid_schemes: 1242 invalid_url = scheme + '://' + schemeless_url 1243 self.assertRaises(urllib.error.HTTPError, h.http_error_302, 1244 req, MockFile(), 302, "Security Loophole", 1245 MockHeaders({"location": invalid_url})) 1246 1247 for scheme in valid_schemes: 1248 valid_url = scheme + '://' + schemeless_url 1249 h.http_error_302(req, MockFile(), 302, "That's fine", 1250 MockHeaders({"location": valid_url})) 1251 self.assertEqual(o.req.get_full_url(), valid_url) 1252 1253 def test_relative_redirect(self): 1254 from_url = "http://example.com/a.html" 1255 relative_url = "/b.html" 1256 h = urllib.request.HTTPRedirectHandler() 1257 o = h.parent = MockOpener() 1258 req = Request(from_url) 1259 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1260 1261 valid_url = urllib.parse.urljoin(from_url,relative_url) 1262 h.http_error_302(req, MockFile(), 302, "That's fine", 1263 MockHeaders({"location": valid_url})) 1264 self.assertEqual(o.req.get_full_url(), valid_url) 1265 1266 def test_cookie_redirect(self): 1267 # cookies shouldn't leak into redirected requests 1268 from http.cookiejar import CookieJar 1269 from test.test_http_cookiejar import interact_netscape 1270 1271 cj = CookieJar() 1272 interact_netscape(cj, "http://www.example.com/", "spam=eggs") 1273 hh = MockHTTPHandler(302, "Location: http://www.cracker.com/\r\n\r\n") 1274 hdeh = urllib.request.HTTPDefaultErrorHandler() 1275 hrh = urllib.request.HTTPRedirectHandler() 1276 cp = urllib.request.HTTPCookieProcessor(cj) 1277 o = build_test_opener(hh, hdeh, hrh, cp) 1278 o.open("http://www.example.com/") 1279 self.assertFalse(hh.req.has_header("Cookie")) 1280 1281 def test_redirect_fragment(self): 1282 redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' 1283 hh = MockHTTPHandler(302, 'Location: ' + redirected_url) 1284 hdeh = urllib.request.HTTPDefaultErrorHandler() 1285 hrh = urllib.request.HTTPRedirectHandler() 1286 o = build_test_opener(hh, hdeh, hrh) 1287 fp = o.open('http://www.example.com') 1288 self.assertEqual(fp.geturl(), redirected_url.strip()) 1289 1290 def test_redirect_no_path(self): 1291 # Issue 14132: Relative redirect strips original path 1292 1293 # clear _opener global variable 1294 self.addCleanup(urllib.request.urlcleanup) 1295 1296 real_class = http.client.HTTPConnection 1297 response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" 1298 http.client.HTTPConnection = test_urllib.fakehttp(response1) 1299 self.addCleanup(setattr, http.client, "HTTPConnection", real_class) 1300 urls = iter(("/path", "/path?query")) 1301 def request(conn, method, url, *pos, **kw): 1302 self.assertEqual(url, next(urls)) 1303 real_class.request(conn, method, url, *pos, **kw) 1304 # Change response for subsequent connection 1305 conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" 1306 http.client.HTTPConnection.request = request 1307 fp = urllib.request.urlopen("http://python.org/path") 1308 self.assertEqual(fp.geturl(), "http://python.org/path?query") 1309 1310 def test_redirect_encoding(self): 1311 # Some characters in the redirect target may need special handling, 1312 # but most ASCII characters should be treated as already encoded 1313 class Handler(urllib.request.HTTPHandler): 1314 def http_open(self, req): 1315 result = self.do_open(self.connection, req) 1316 self.last_buf = self.connection.buf 1317 # Set up a normal response for the next request 1318 self.connection = test_urllib.fakehttp( 1319 b'HTTP/1.1 200 OK\r\n' 1320 b'Content-Length: 3\r\n' 1321 b'\r\n' 1322 b'123' 1323 ) 1324 return result 1325 handler = Handler() 1326 opener = urllib.request.build_opener(handler) 1327 tests = ( 1328 (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), 1329 (b'/spaced%20path/', b'/spaced%20path/'), 1330 (b'/spaced path/', b'/spaced%20path/'), 1331 (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), 1332 ) 1333 for [location, result] in tests: 1334 with self.subTest(repr(location)): 1335 handler.connection = test_urllib.fakehttp( 1336 b'HTTP/1.1 302 Redirect\r\n' 1337 b'Location: ' + location + b'\r\n' 1338 b'\r\n' 1339 ) 1340 response = opener.open('http://example.com/') 1341 expected = b'GET ' + result + b' ' 1342 request = handler.last_buf 1343 self.assertTrue(request.startswith(expected), repr(request)) 1344 1345 def test_proxy(self): 1346 u = "proxy.example.com:3128" 1347 for d in dict(http=u), dict(HTTP=u): 1348 o = OpenerDirector() 1349 ph = urllib.request.ProxyHandler(d) 1350 o.add_handler(ph) 1351 meth_spec = [ 1352 [("http_open", "return response")] 1353 ] 1354 handlers = add_ordered_mock_handlers(o, meth_spec) 1355 1356 req = Request("http://acme.example.com/") 1357 self.assertEqual(req.host, "acme.example.com") 1358 o.open(req) 1359 self.assertEqual(req.host, u) 1360 self.assertEqual([(handlers[0], "http_open")], 1361 [tup[0:2] for tup in o.calls]) 1362 1363 def test_proxy_no_proxy(self): 1364 os.environ['no_proxy'] = 'python.org' 1365 o = OpenerDirector() 1366 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1367 o.add_handler(ph) 1368 req = Request("http://www.perl.org/") 1369 self.assertEqual(req.host, "www.perl.org") 1370 o.open(req) 1371 self.assertEqual(req.host, "proxy.example.com") 1372 req = Request("http://www.python.org") 1373 self.assertEqual(req.host, "www.python.org") 1374 o.open(req) 1375 self.assertEqual(req.host, "www.python.org") 1376 del os.environ['no_proxy'] 1377 1378 def test_proxy_no_proxy_all(self): 1379 os.environ['no_proxy'] = '*' 1380 o = OpenerDirector() 1381 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1382 o.add_handler(ph) 1383 req = Request("http://www.python.org") 1384 self.assertEqual(req.host, "www.python.org") 1385 o.open(req) 1386 self.assertEqual(req.host, "www.python.org") 1387 del os.environ['no_proxy'] 1388 1389 def test_proxy_https(self): 1390 o = OpenerDirector() 1391 ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) 1392 o.add_handler(ph) 1393 meth_spec = [ 1394 [("https_open", "return response")] 1395 ] 1396 handlers = add_ordered_mock_handlers(o, meth_spec) 1397 1398 req = Request("https://www.example.com/") 1399 self.assertEqual(req.host, "www.example.com") 1400 o.open(req) 1401 self.assertEqual(req.host, "proxy.example.com:3128") 1402 self.assertEqual([(handlers[0], "https_open")], 1403 [tup[0:2] for tup in o.calls]) 1404 1405 def test_proxy_https_proxy_authorization(self): 1406 o = OpenerDirector() 1407 ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) 1408 o.add_handler(ph) 1409 https_handler = MockHTTPSHandler() 1410 o.add_handler(https_handler) 1411 req = Request("https://www.example.com/") 1412 req.add_header("Proxy-Authorization", "FooBar") 1413 req.add_header("User-Agent", "Grail") 1414 self.assertEqual(req.host, "www.example.com") 1415 self.assertIsNone(req._tunnel_host) 1416 o.open(req) 1417 # Verify Proxy-Authorization gets tunneled to request. 1418 # httpsconn req_headers do not have the Proxy-Authorization header but 1419 # the req will have. 1420 self.assertNotIn(("Proxy-Authorization", "FooBar"), 1421 https_handler.httpconn.req_headers) 1422 self.assertIn(("User-Agent", "Grail"), 1423 https_handler.httpconn.req_headers) 1424 self.assertIsNotNone(req._tunnel_host) 1425 self.assertEqual(req.host, "proxy.example.com:3128") 1426 self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") 1427 1428 @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") 1429 def test_osx_proxy_bypass(self): 1430 bypass = { 1431 'exclude_simple': False, 1432 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', 1433 '10.0/16'] 1434 } 1435 # Check hosts that should trigger the proxy bypass 1436 for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', 1437 '10.0.0.1'): 1438 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), 1439 'expected bypass of %s to be True' % host) 1440 # Check hosts that should not trigger the proxy bypass 1441 for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', 1442 'notinbypass'): 1443 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), 1444 'expected bypass of %s to be False' % host) 1445 1446 # Check the exclude_simple flag 1447 bypass = {'exclude_simple': True, 'exceptions': []} 1448 self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) 1449 1450 # Check that invalid prefix lengths are ignored 1451 bypass = { 1452 'exclude_simple': False, 1453 'exceptions': [ '10.0.0.0/40', '172.19.10.0/24' ] 1454 } 1455 host = '172.19.10.5' 1456 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), 1457 'expected bypass of %s to be True' % host) 1458 host = '10.0.1.5' 1459 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), 1460 'expected bypass of %s to be False' % host) 1461 1462 def check_basic_auth(self, headers, realm): 1463 with self.subTest(realm=realm, headers=headers): 1464 opener = OpenerDirector() 1465 password_manager = MockPasswordManager() 1466 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) 1467 body = '\r\n'.join(headers) + '\r\n\r\n' 1468 http_handler = MockHTTPHandler(401, body) 1469 opener.add_handler(auth_handler) 1470 opener.add_handler(http_handler) 1471 self._test_basic_auth(opener, auth_handler, "Authorization", 1472 realm, http_handler, password_manager, 1473 "http://acme.example.com/protected", 1474 "http://acme.example.com/protected") 1475 1476 def test_basic_auth(self): 1477 realm = "realm2@example.com" 1478 realm2 = "realm2@example.com" 1479 basic = f'Basic realm="{realm}"' 1480 basic2 = f'Basic realm="{realm2}"' 1481 other_no_realm = 'Otherscheme xxx' 1482 digest = (f'Digest realm="{realm2}", ' 1483 f'qop="auth, auth-int", ' 1484 f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", ' 1485 f'opaque="5ccc069c403ebaf9f0171e9517f40e41"') 1486 for realm_str in ( 1487 # test "quote" and 'quote' 1488 f'Basic realm="{realm}"', 1489 f"Basic realm='{realm}'", 1490 1491 # charset is ignored 1492 f'Basic realm="{realm}", charset="UTF-8"', 1493 1494 # Multiple challenges per header 1495 f'{basic}, {basic2}', 1496 f'{basic}, {other_no_realm}', 1497 f'{other_no_realm}, {basic}', 1498 f'{basic}, {digest}', 1499 f'{digest}, {basic}', 1500 ): 1501 headers = [f'WWW-Authenticate: {realm_str}'] 1502 self.check_basic_auth(headers, realm) 1503 1504 # no quote: expect a warning 1505 with support.check_warnings(("Basic Auth Realm was unquoted", 1506 UserWarning)): 1507 headers = [f'WWW-Authenticate: Basic realm={realm}'] 1508 self.check_basic_auth(headers, realm) 1509 1510 # Multiple headers: one challenge per header. 1511 # Use the first Basic realm. 1512 for challenges in ( 1513 [basic, basic2], 1514 [basic, digest], 1515 [digest, basic], 1516 ): 1517 headers = [f'WWW-Authenticate: {challenge}' 1518 for challenge in challenges] 1519 self.check_basic_auth(headers, realm) 1520 1521 def test_proxy_basic_auth(self): 1522 opener = OpenerDirector() 1523 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) 1524 opener.add_handler(ph) 1525 password_manager = MockPasswordManager() 1526 auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) 1527 realm = "ACME Networks" 1528 http_handler = MockHTTPHandler( 1529 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1530 opener.add_handler(auth_handler) 1531 opener.add_handler(http_handler) 1532 self._test_basic_auth(opener, auth_handler, "Proxy-authorization", 1533 realm, http_handler, password_manager, 1534 "http://acme.example.com:3128/protected", 1535 "proxy.example.com:3128", 1536 ) 1537 1538 def test_basic_and_digest_auth_handlers(self): 1539 # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* 1540 # response (http://python.org/sf/1479302), where it should instead 1541 # return None to allow another handler (especially 1542 # HTTPBasicAuthHandler) to handle the response. 1543 1544 # Also (http://python.org/sf/14797027, RFC 2617 section 1.2), we must 1545 # try digest first (since it's the strongest auth scheme), so we record 1546 # order of calls here to check digest comes first: 1547 class RecordingOpenerDirector(OpenerDirector): 1548 def __init__(self): 1549 OpenerDirector.__init__(self) 1550 self.recorded = [] 1551 1552 def record(self, info): 1553 self.recorded.append(info) 1554 1555 class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): 1556 def http_error_401(self, *args, **kwds): 1557 self.parent.record("digest") 1558 urllib.request.HTTPDigestAuthHandler.http_error_401(self, 1559 *args, **kwds) 1560 1561 class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): 1562 def http_error_401(self, *args, **kwds): 1563 self.parent.record("basic") 1564 urllib.request.HTTPBasicAuthHandler.http_error_401(self, 1565 *args, **kwds) 1566 1567 opener = RecordingOpenerDirector() 1568 password_manager = MockPasswordManager() 1569 digest_handler = TestDigestAuthHandler(password_manager) 1570 basic_handler = TestBasicAuthHandler(password_manager) 1571 realm = "ACME Networks" 1572 http_handler = MockHTTPHandler( 1573 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1574 opener.add_handler(basic_handler) 1575 opener.add_handler(digest_handler) 1576 opener.add_handler(http_handler) 1577 1578 # check basic auth isn't blocked by digest handler failing 1579 self._test_basic_auth(opener, basic_handler, "Authorization", 1580 realm, http_handler, password_manager, 1581 "http://acme.example.com/protected", 1582 "http://acme.example.com/protected", 1583 ) 1584 # check digest was tried before basic (twice, because 1585 # _test_basic_auth called .open() twice) 1586 self.assertEqual(opener.recorded, ["digest", "basic"]*2) 1587 1588 def test_unsupported_auth_digest_handler(self): 1589 opener = OpenerDirector() 1590 # While using DigestAuthHandler 1591 digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) 1592 http_handler = MockHTTPHandler( 1593 401, 'WWW-Authenticate: Kerberos\r\n\r\n') 1594 opener.add_handler(digest_auth_handler) 1595 opener.add_handler(http_handler) 1596 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1597 1598 def test_unsupported_auth_basic_handler(self): 1599 # While using BasicAuthHandler 1600 opener = OpenerDirector() 1601 basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) 1602 http_handler = MockHTTPHandler( 1603 401, 'WWW-Authenticate: NTLM\r\n\r\n') 1604 opener.add_handler(basic_auth_handler) 1605 opener.add_handler(http_handler) 1606 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1607 1608 def _test_basic_auth(self, opener, auth_handler, auth_header, 1609 realm, http_handler, password_manager, 1610 request_url, protected_url): 1611 import base64 1612 user, password = "wile", "coyote" 1613 1614 # .add_password() fed through to password manager 1615 auth_handler.add_password(realm, request_url, user, password) 1616 self.assertEqual(realm, password_manager.realm) 1617 self.assertEqual(request_url, password_manager.url) 1618 self.assertEqual(user, password_manager.user) 1619 self.assertEqual(password, password_manager.password) 1620 1621 opener.open(request_url) 1622 1623 # should have asked the password manager for the username/password 1624 self.assertEqual(password_manager.target_realm, realm) 1625 self.assertEqual(password_manager.target_url, protected_url) 1626 1627 # expect one request without authorization, then one with 1628 self.assertEqual(len(http_handler.requests), 2) 1629 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1630 userpass = bytes('%s:%s' % (user, password), "ascii") 1631 auth_hdr_value = ('Basic ' + 1632 base64.encodebytes(userpass).strip().decode()) 1633 self.assertEqual(http_handler.requests[1].get_header(auth_header), 1634 auth_hdr_value) 1635 self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], 1636 auth_hdr_value) 1637 # if the password manager can't find a password, the handler won't 1638 # handle the HTTP auth error 1639 password_manager.user = password_manager.password = None 1640 http_handler.reset() 1641 opener.open(request_url) 1642 self.assertEqual(len(http_handler.requests), 1) 1643 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1644 1645 def test_basic_prior_auth_auto_send(self): 1646 # Assume already authenticated if is_authenticated=True 1647 # for APIs like Github that don't return 401 1648 1649 user, password = "wile", "coyote" 1650 request_url = "http://acme.example.com/protected" 1651 1652 http_handler = MockHTTPHandlerCheckAuth(200) 1653 1654 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1655 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1656 auth_prior_handler.add_password( 1657 None, request_url, user, password, is_authenticated=True) 1658 1659 is_auth = pwd_manager.is_authenticated(request_url) 1660 self.assertTrue(is_auth) 1661 1662 opener = OpenerDirector() 1663 opener.add_handler(auth_prior_handler) 1664 opener.add_handler(http_handler) 1665 1666 opener.open(request_url) 1667 1668 # expect request to be sent with auth header 1669 self.assertTrue(http_handler.has_auth_header) 1670 1671 def test_basic_prior_auth_send_after_first_success(self): 1672 # Auto send auth header after authentication is successful once 1673 1674 user, password = 'wile', 'coyote' 1675 request_url = 'http://acme.example.com/protected' 1676 realm = 'ACME' 1677 1678 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1679 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1680 auth_prior_handler.add_password(realm, request_url, user, password) 1681 1682 is_auth = pwd_manager.is_authenticated(request_url) 1683 self.assertFalse(is_auth) 1684 1685 opener = OpenerDirector() 1686 opener.add_handler(auth_prior_handler) 1687 1688 http_handler = MockHTTPHandler( 1689 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) 1690 opener.add_handler(http_handler) 1691 1692 opener.open(request_url) 1693 1694 is_auth = pwd_manager.is_authenticated(request_url) 1695 self.assertTrue(is_auth) 1696 1697 http_handler = MockHTTPHandlerCheckAuth(200) 1698 self.assertFalse(http_handler.has_auth_header) 1699 1700 opener = OpenerDirector() 1701 opener.add_handler(auth_prior_handler) 1702 opener.add_handler(http_handler) 1703 1704 # After getting 200 from MockHTTPHandler 1705 # Next request sends header in the first request 1706 opener.open(request_url) 1707 1708 # expect request to be sent with auth header 1709 self.assertTrue(http_handler.has_auth_header) 1710 1711 def test_http_closed(self): 1712 """Test the connection is cleaned up when the response is closed""" 1713 for (transfer, data) in ( 1714 ("Connection: close", b"data"), 1715 ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), 1716 ("Content-Length: 4", b"data"), 1717 ): 1718 header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) 1719 conn = test_urllib.fakehttp(header.encode() + data) 1720 handler = urllib.request.AbstractHTTPHandler() 1721 req = Request("http://dummy/") 1722 req.timeout = None 1723 with handler.do_open(conn, req) as resp: 1724 resp.read() 1725 self.assertTrue(conn.fakesock.closed, 1726 "Connection not closed with {!r}".format(transfer)) 1727 1728 def test_invalid_closed(self): 1729 """Test the connection is cleaned up after an invalid response""" 1730 conn = test_urllib.fakehttp(b"") 1731 handler = urllib.request.AbstractHTTPHandler() 1732 req = Request("http://dummy/") 1733 req.timeout = None 1734 with self.assertRaises(http.client.BadStatusLine): 1735 handler.do_open(conn, req) 1736 self.assertTrue(conn.fakesock.closed, "Connection not closed") 1737 1738 1739class MiscTests(unittest.TestCase): 1740 1741 def opener_has_handler(self, opener, handler_class): 1742 self.assertTrue(any(h.__class__ == handler_class 1743 for h in opener.handlers)) 1744 1745 def test_build_opener(self): 1746 class MyHTTPHandler(urllib.request.HTTPHandler): 1747 pass 1748 1749 class FooHandler(urllib.request.BaseHandler): 1750 def foo_open(self): 1751 pass 1752 1753 class BarHandler(urllib.request.BaseHandler): 1754 def bar_open(self): 1755 pass 1756 1757 build_opener = urllib.request.build_opener 1758 1759 o = build_opener(FooHandler, BarHandler) 1760 self.opener_has_handler(o, FooHandler) 1761 self.opener_has_handler(o, BarHandler) 1762 1763 # can take a mix of classes and instances 1764 o = build_opener(FooHandler, BarHandler()) 1765 self.opener_has_handler(o, FooHandler) 1766 self.opener_has_handler(o, BarHandler) 1767 1768 # subclasses of default handlers override default handlers 1769 o = build_opener(MyHTTPHandler) 1770 self.opener_has_handler(o, MyHTTPHandler) 1771 1772 # a particular case of overriding: default handlers can be passed 1773 # in explicitly 1774 o = build_opener() 1775 self.opener_has_handler(o, urllib.request.HTTPHandler) 1776 o = build_opener(urllib.request.HTTPHandler) 1777 self.opener_has_handler(o, urllib.request.HTTPHandler) 1778 o = build_opener(urllib.request.HTTPHandler()) 1779 self.opener_has_handler(o, urllib.request.HTTPHandler) 1780 1781 # Issue2670: multiple handlers sharing the same base class 1782 class MyOtherHTTPHandler(urllib.request.HTTPHandler): 1783 pass 1784 1785 o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) 1786 self.opener_has_handler(o, MyHTTPHandler) 1787 self.opener_has_handler(o, MyOtherHTTPHandler) 1788 1789 @unittest.skipUnless(support.is_resource_enabled('network'), 1790 'test requires network access') 1791 def test_issue16464(self): 1792 with socket_helper.transient_internet("http://www.example.com/"): 1793 opener = urllib.request.build_opener() 1794 request = urllib.request.Request("http://www.example.com/") 1795 self.assertEqual(None, request.data) 1796 1797 opener.open(request, "1".encode("us-ascii")) 1798 self.assertEqual(b"1", request.data) 1799 self.assertEqual("1", request.get_header("Content-length")) 1800 1801 opener.open(request, "1234567890".encode("us-ascii")) 1802 self.assertEqual(b"1234567890", request.data) 1803 self.assertEqual("10", request.get_header("Content-length")) 1804 1805 def test_HTTPError_interface(self): 1806 """ 1807 Issue 13211 reveals that HTTPError didn't implement the URLError 1808 interface even though HTTPError is a subclass of URLError. 1809 """ 1810 msg = 'something bad happened' 1811 url = code = fp = None 1812 hdrs = 'Content-Length: 42' 1813 err = urllib.error.HTTPError(url, code, msg, hdrs, fp) 1814 self.assertTrue(hasattr(err, 'reason')) 1815 self.assertEqual(err.reason, 'something bad happened') 1816 self.assertTrue(hasattr(err, 'headers')) 1817 self.assertEqual(err.headers, 'Content-Length: 42') 1818 expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) 1819 self.assertEqual(str(err), expected_errmsg) 1820 expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) 1821 self.assertEqual(repr(err), expected_errmsg) 1822 1823 def test_parse_proxy(self): 1824 parse_proxy_test_cases = [ 1825 ('proxy.example.com', 1826 (None, None, None, 'proxy.example.com')), 1827 ('proxy.example.com:3128', 1828 (None, None, None, 'proxy.example.com:3128')), 1829 ('proxy.example.com', (None, None, None, 'proxy.example.com')), 1830 ('proxy.example.com:3128', 1831 (None, None, None, 'proxy.example.com:3128')), 1832 # The authority component may optionally include userinfo 1833 # (assumed to be # username:password): 1834 ('joe:password@proxy.example.com', 1835 (None, 'joe', 'password', 'proxy.example.com')), 1836 ('joe:password@proxy.example.com:3128', 1837 (None, 'joe', 'password', 'proxy.example.com:3128')), 1838 #Examples with URLS 1839 ('http://proxy.example.com/', 1840 ('http', None, None, 'proxy.example.com')), 1841 ('http://proxy.example.com:3128/', 1842 ('http', None, None, 'proxy.example.com:3128')), 1843 ('http://joe:password@proxy.example.com/', 1844 ('http', 'joe', 'password', 'proxy.example.com')), 1845 ('http://joe:password@proxy.example.com:3128', 1846 ('http', 'joe', 'password', 'proxy.example.com:3128')), 1847 # Everything after the authority is ignored 1848 ('ftp://joe:password@proxy.example.com/rubbish:3128', 1849 ('ftp', 'joe', 'password', 'proxy.example.com')), 1850 # Test for no trailing '/' case 1851 ('http://joe:password@proxy.example.com', 1852 ('http', 'joe', 'password', 'proxy.example.com')), 1853 # Testcases with '/' character in username, password 1854 ('http://user/name:password@localhost:22', 1855 ('http', 'user/name', 'password', 'localhost:22')), 1856 ('http://username:pass/word@localhost:22', 1857 ('http', 'username', 'pass/word', 'localhost:22')), 1858 ('http://user/name:pass/word@localhost:22', 1859 ('http', 'user/name', 'pass/word', 'localhost:22')), 1860 ] 1861 1862 1863 for tc, expected in parse_proxy_test_cases: 1864 self.assertEqual(_parse_proxy(tc), expected) 1865 1866 self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), 1867 1868 def test_unsupported_algorithm(self): 1869 handler = AbstractDigestAuthHandler() 1870 with self.assertRaises(ValueError) as exc: 1871 handler.get_algorithm_impls('invalid') 1872 self.assertEqual( 1873 str(exc.exception), 1874 "Unsupported digest authentication algorithm 'invalid'" 1875 ) 1876 1877 1878class RequestTests(unittest.TestCase): 1879 class PutRequest(Request): 1880 method = 'PUT' 1881 1882 def setUp(self): 1883 self.get = Request("http://www.python.org/~jeremy/") 1884 self.post = Request("http://www.python.org/~jeremy/", 1885 "data", 1886 headers={"X-Test": "test"}) 1887 self.head = Request("http://www.python.org/~jeremy/", method='HEAD') 1888 self.put = self.PutRequest("http://www.python.org/~jeremy/") 1889 self.force_post = self.PutRequest("http://www.python.org/~jeremy/", 1890 method="POST") 1891 1892 def test_method(self): 1893 self.assertEqual("POST", self.post.get_method()) 1894 self.assertEqual("GET", self.get.get_method()) 1895 self.assertEqual("HEAD", self.head.get_method()) 1896 self.assertEqual("PUT", self.put.get_method()) 1897 self.assertEqual("POST", self.force_post.get_method()) 1898 1899 def test_data(self): 1900 self.assertFalse(self.get.data) 1901 self.assertEqual("GET", self.get.get_method()) 1902 self.get.data = "spam" 1903 self.assertTrue(self.get.data) 1904 self.assertEqual("POST", self.get.get_method()) 1905 1906 # issue 16464 1907 # if we change data we need to remove content-length header 1908 # (cause it's most probably calculated for previous value) 1909 def test_setting_data_should_remove_content_length(self): 1910 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1911 self.get.add_unredirected_header("Content-length", 42) 1912 self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) 1913 self.get.data = "spam" 1914 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1915 1916 # issue 17485 same for deleting data. 1917 def test_deleting_data_should_remove_content_length(self): 1918 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1919 self.get.data = 'foo' 1920 self.get.add_unredirected_header("Content-length", 3) 1921 self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) 1922 del self.get.data 1923 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1924 1925 def test_get_full_url(self): 1926 self.assertEqual("http://www.python.org/~jeremy/", 1927 self.get.get_full_url()) 1928 1929 def test_selector(self): 1930 self.assertEqual("/~jeremy/", self.get.selector) 1931 req = Request("http://www.python.org/") 1932 self.assertEqual("/", req.selector) 1933 1934 def test_get_type(self): 1935 self.assertEqual("http", self.get.type) 1936 1937 def test_get_host(self): 1938 self.assertEqual("www.python.org", self.get.host) 1939 1940 def test_get_host_unquote(self): 1941 req = Request("http://www.%70ython.org/") 1942 self.assertEqual("www.python.org", req.host) 1943 1944 def test_proxy(self): 1945 self.assertFalse(self.get.has_proxy()) 1946 self.get.set_proxy("www.perl.org", "http") 1947 self.assertTrue(self.get.has_proxy()) 1948 self.assertEqual("www.python.org", self.get.origin_req_host) 1949 self.assertEqual("www.perl.org", self.get.host) 1950 1951 def test_wrapped_url(self): 1952 req = Request("<URL:http://www.python.org>") 1953 self.assertEqual("www.python.org", req.host) 1954 1955 def test_url_fragment(self): 1956 req = Request("http://www.python.org/?qs=query#fragment=true") 1957 self.assertEqual("/?qs=query", req.selector) 1958 req = Request("http://www.python.org/#fun=true") 1959 self.assertEqual("/", req.selector) 1960 1961 # Issue 11703: geturl() omits fragment in the original URL. 1962 url = 'http://docs.python.org/library/urllib2.html#OK' 1963 req = Request(url) 1964 self.assertEqual(req.get_full_url(), url) 1965 1966 def test_url_fullurl_get_full_url(self): 1967 urls = ['http://docs.python.org', 1968 'http://docs.python.org/library/urllib2.html#OK', 1969 'http://www.python.org/?qs=query#fragment=true'] 1970 for url in urls: 1971 req = Request(url) 1972 self.assertEqual(req.get_full_url(), req.full_url) 1973 1974 1975if __name__ == "__main__": 1976 unittest.main() 1977
