1import unittest 2from test import support 3from test.support import os_helper 4from test.support import requires_subprocess 5from test.support import warnings_helper 6from test import test_urllib 7from unittest import mock 8 9import os 10import io 11import socket 12import array 13import sys 14import tempfile 15import subprocess 16 17import urllib.request 18# The proxy bypass method imported below has logic specific to the 19# corresponding system but is testable on all platforms. 20from urllib.request import (Request, OpenerDirector, HTTPBasicAuthHandler, 21 HTTPPasswordMgrWithPriorAuth, _parse_proxy, 22 _proxy_bypass_winreg_override, 23 _proxy_bypass_macosx_sysconf, 24 AbstractDigestAuthHandler) 25from urllib.parse import urlparse 26import urllib.error 27import http.client 28 29support.requires_working_socket(module=True) 30 31# XXX 32# Request 33# CacheFTPHandler (hard to write) 34# parse_keqv_list, parse_http_list, HTTPDigestAuthHandler 35 36 37class TrivialTests(unittest.TestCase): 38 39 def test___all__(self): 40 # Verify which names are exposed 41 for module in 'request', 'response', 'parse', 'error', 'robotparser': 42 context = {} 43 exec('from urllib.%s import *' % module, context) 44 del context['__builtins__'] 45 if module == 'request' and os.name == 'nt': 46 u, p = context.pop('url2pathname'), context.pop('pathname2url') 47 self.assertEqual(u.__module__, 'nturl2path') 48 self.assertEqual(p.__module__, 'nturl2path') 49 for k, v in context.items(): 50 self.assertEqual(v.__module__, 'urllib.%s' % module, 51 "%r is exposed in 'urllib.%s' but defined in %r" % 52 (k, module, v.__module__)) 53 54 def test_trivial(self): 55 # A couple trivial tests 56 57 # clear _opener global variable 58 self.addCleanup(urllib.request.urlcleanup) 59 60 self.assertRaises(ValueError, urllib.request.urlopen, 'bogus url') 61 62 # XXX Name hacking to get this to work on Windows. 63 fname = os.path.abspath(urllib.request.__file__).replace(os.sep, '/') 64 65 if os.name == 'nt': 66 file_url = "file:///%s" % fname 67 else: 68 file_url = "file://%s" % fname 69 70 with urllib.request.urlopen(file_url) as f: 71 f.read() 72 73 def test_parse_http_list(self): 74 tests = [ 75 ('a,b,c', ['a', 'b', 'c']), 76 ('path"o,l"og"i"cal, example', ['path"o,l"og"i"cal', 'example']), 77 ('a, b, "c", "d", "e,f", g, h', 78 ['a', 'b', '"c"', '"d"', '"e,f"', 'g', 'h']), 79 ('a="b\\"c", d="e\\,f", g="h\\\\i"', 80 ['a="b"c"', 'd="e,f"', 'g="h\\i"'])] 81 for string, list in tests: 82 self.assertEqual(urllib.request.parse_http_list(string), list) 83 84 def test_URLError_reasonstr(self): 85 err = urllib.error.URLError('reason') 86 self.assertIn(err.reason, str(err)) 87 88 89class RequestHdrsTests(unittest.TestCase): 90 91 def test_request_headers_dict(self): 92 """ 93 The Request.headers dictionary is not a documented interface. It 94 should stay that way, because the complete set of headers are only 95 accessible through the .get_header(), .has_header(), .header_items() 96 interface. However, .headers pre-dates those methods, and so real code 97 will be using the dictionary. 98 99 The introduction in 2.4 of those methods was a mistake for the same 100 reason: code that previously saw all (urllib2 user)-provided headers in 101 .headers now sees only a subset. 102 103 """ 104 url = "http://example.com" 105 self.assertEqual(Request(url, 106 headers={"Spam-eggs": "blah"} 107 ).headers["Spam-eggs"], "blah") 108 self.assertEqual(Request(url, 109 headers={"spam-EggS": "blah"} 110 ).headers["Spam-eggs"], "blah") 111 112 def test_request_headers_methods(self): 113 """ 114 Note the case normalization of header names here, to 115 .capitalize()-case. This should be preserved for 116 backwards-compatibility. (In the HTTP case, normalization to 117 .title()-case is done by urllib2 before sending headers to 118 http.client). 119 120 Note that e.g. r.has_header("spam-EggS") is currently False, and 121 r.get_header("spam-EggS") returns None, but that could be changed in 122 future. 123 124 Method r.remove_header should remove items both from r.headers and 125 r.unredirected_hdrs dictionaries 126 """ 127 url = "http://example.com" 128 req = Request(url, headers={"Spam-eggs": "blah"}) 129 self.assertTrue(req.has_header("Spam-eggs")) 130 self.assertEqual(req.header_items(), [('Spam-eggs', 'blah')]) 131 132 req.add_header("Foo-Bar", "baz") 133 self.assertEqual(sorted(req.header_items()), 134 [('Foo-bar', 'baz'), ('Spam-eggs', 'blah')]) 135 self.assertFalse(req.has_header("Not-there")) 136 self.assertIsNone(req.get_header("Not-there")) 137 self.assertEqual(req.get_header("Not-there", "default"), "default") 138 139 req.remove_header("Spam-eggs") 140 self.assertFalse(req.has_header("Spam-eggs")) 141 142 req.add_unredirected_header("Unredirected-spam", "Eggs") 143 self.assertTrue(req.has_header("Unredirected-spam")) 144 145 req.remove_header("Unredirected-spam") 146 self.assertFalse(req.has_header("Unredirected-spam")) 147 148 def test_password_manager(self): 149 mgr = urllib.request.HTTPPasswordMgr() 150 add = mgr.add_password 151 find_user_pass = mgr.find_user_password 152 153 add("Some Realm", "http://example.com/", "joe", "password") 154 add("Some Realm", "http://example.com/ni", "ni", "ni") 155 add("Some Realm", "http://c.example.com:3128", "3", "c") 156 add("Some Realm", "d.example.com", "4", "d") 157 add("Some Realm", "e.example.com:3128", "5", "e") 158 159 # For the same realm, password set the highest path is the winner. 160 self.assertEqual(find_user_pass("Some Realm", "example.com"), 161 ('joe', 'password')) 162 self.assertEqual(find_user_pass("Some Realm", "http://example.com/ni"), 163 ('joe', 'password')) 164 self.assertEqual(find_user_pass("Some Realm", "http://example.com"), 165 ('joe', 'password')) 166 self.assertEqual(find_user_pass("Some Realm", "http://example.com/"), 167 ('joe', 'password')) 168 self.assertEqual(find_user_pass("Some Realm", 169 "http://example.com/spam"), 170 ('joe', 'password')) 171 self.assertEqual(find_user_pass("Some Realm", 172 "http://example.com/spam/spam"), 173 ('joe', 'password')) 174 175 # You can have different passwords for different paths. 176 177 add("c", "http://example.com/foo", "foo", "ni") 178 add("c", "http://example.com/bar", "bar", "nini") 179 add("c", "http://example.com/foo/bar", "foobar", "nibar") 180 181 self.assertEqual(find_user_pass("c", "http://example.com/foo"), 182 ('foo', 'ni')) 183 self.assertEqual(find_user_pass("c", "http://example.com/bar"), 184 ('bar', 'nini')) 185 self.assertEqual(find_user_pass("c", "http://example.com/foo/"), 186 ('foo', 'ni')) 187 self.assertEqual(find_user_pass("c", "http://example.com/foo/bar"), 188 ('foo', 'ni')) 189 self.assertEqual(find_user_pass("c", "http://example.com/foo/baz"), 190 ('foo', 'ni')) 191 self.assertEqual(find_user_pass("c", "http://example.com/foobar"), 192 (None, None)) 193 194 add("c", "http://example.com/baz/", "baz", "ninini") 195 196 self.assertEqual(find_user_pass("c", "http://example.com/baz"), 197 (None, None)) 198 self.assertEqual(find_user_pass("c", "http://example.com/baz/"), 199 ('baz', 'ninini')) 200 self.assertEqual(find_user_pass("c", "http://example.com/baz/bar"), 201 ('baz', 'ninini')) 202 203 # For the same path, newer password should be considered. 204 205 add("b", "http://example.com/", "first", "blah") 206 add("b", "http://example.com/", "second", "spam") 207 208 self.assertEqual(find_user_pass("b", "http://example.com/"), 209 ('second', 'spam')) 210 211 # No special relationship between a.example.com and example.com: 212 213 add("a", "http://example.com", "1", "a") 214 self.assertEqual(find_user_pass("a", "http://example.com/"), 215 ('1', 'a')) 216 217 self.assertEqual(find_user_pass("a", "http://a.example.com/"), 218 (None, None)) 219 220 # Ports: 221 222 self.assertEqual(find_user_pass("Some Realm", "c.example.com"), 223 (None, None)) 224 self.assertEqual(find_user_pass("Some Realm", "c.example.com:3128"), 225 ('3', 'c')) 226 self.assertEqual( 227 find_user_pass("Some Realm", "http://c.example.com:3128"), 228 ('3', 'c')) 229 self.assertEqual(find_user_pass("Some Realm", "d.example.com"), 230 ('4', 'd')) 231 self.assertEqual(find_user_pass("Some Realm", "e.example.com:3128"), 232 ('5', 'e')) 233 234 def test_password_manager_default_port(self): 235 """ 236 The point to note here is that we can't guess the default port if 237 there's no scheme. This applies to both add_password and 238 find_user_password. 239 """ 240 mgr = urllib.request.HTTPPasswordMgr() 241 add = mgr.add_password 242 find_user_pass = mgr.find_user_password 243 add("f", "http://g.example.com:80", "10", "j") 244 add("g", "http://h.example.com", "11", "k") 245 add("h", "i.example.com:80", "12", "l") 246 add("i", "j.example.com", "13", "m") 247 self.assertEqual(find_user_pass("f", "g.example.com:100"), 248 (None, None)) 249 self.assertEqual(find_user_pass("f", "g.example.com:80"), 250 ('10', 'j')) 251 self.assertEqual(find_user_pass("f", "g.example.com"), 252 (None, None)) 253 self.assertEqual(find_user_pass("f", "http://g.example.com:100"), 254 (None, None)) 255 self.assertEqual(find_user_pass("f", "http://g.example.com:80"), 256 ('10', 'j')) 257 self.assertEqual(find_user_pass("f", "http://g.example.com"), 258 ('10', 'j')) 259 self.assertEqual(find_user_pass("g", "h.example.com"), ('11', 'k')) 260 self.assertEqual(find_user_pass("g", "h.example.com:80"), ('11', 'k')) 261 self.assertEqual(find_user_pass("g", "http://h.example.com:80"), 262 ('11', 'k')) 263 self.assertEqual(find_user_pass("h", "i.example.com"), (None, None)) 264 self.assertEqual(find_user_pass("h", "i.example.com:80"), ('12', 'l')) 265 self.assertEqual(find_user_pass("h", "http://i.example.com:80"), 266 ('12', 'l')) 267 self.assertEqual(find_user_pass("i", "j.example.com"), ('13', 'm')) 268 self.assertEqual(find_user_pass("i", "j.example.com:80"), 269 (None, None)) 270 self.assertEqual(find_user_pass("i", "http://j.example.com"), 271 ('13', 'm')) 272 self.assertEqual(find_user_pass("i", "http://j.example.com:80"), 273 (None, None)) 274 275 276class MockOpener: 277 addheaders = [] 278 279 def open(self, req, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 280 self.req, self.data, self.timeout = req, data, timeout 281 282 def error(self, proto, *args): 283 self.proto, self.args = proto, args 284 285 286class MockFile: 287 def read(self, count=None): 288 pass 289 290 def readline(self, count=None): 291 pass 292 293 def close(self): 294 pass 295 296 297class MockHeaders(dict): 298 def getheaders(self, name): 299 return list(self.values()) 300 301 302class MockResponse(io.StringIO): 303 def __init__(self, code, msg, headers, data, url=None): 304 io.StringIO.__init__(self, data) 305 self.code, self.msg, self.headers, self.url = code, msg, headers, url 306 307 def info(self): 308 return self.headers 309 310 def geturl(self): 311 return self.url 312 313 314class MockCookieJar: 315 def add_cookie_header(self, request): 316 self.ach_req = request 317 318 def extract_cookies(self, response, request): 319 self.ec_req, self.ec_r = request, response 320 321 322class FakeMethod: 323 def __init__(self, meth_name, action, handle): 324 self.meth_name = meth_name 325 self.handle = handle 326 self.action = action 327 328 def __call__(self, *args): 329 return self.handle(self.meth_name, self.action, *args) 330 331 332class MockHTTPResponse(io.IOBase): 333 def __init__(self, fp, msg, status, reason): 334 self.fp = fp 335 self.msg = msg 336 self.status = status 337 self.reason = reason 338 self.code = 200 339 340 def read(self): 341 return '' 342 343 def info(self): 344 return {} 345 346 def geturl(self): 347 return self.url 348 349 350class MockHTTPClass: 351 def __init__(self): 352 self.level = 0 353 self.req_headers = [] 354 self.data = None 355 self.raise_on_endheaders = False 356 self.sock = None 357 self._tunnel_headers = {} 358 359 def __call__(self, host, timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 360 self.host = host 361 self.timeout = timeout 362 return self 363 364 def set_debuglevel(self, level): 365 self.level = level 366 367 def set_tunnel(self, host, port=None, headers=None): 368 self._tunnel_host = host 369 self._tunnel_port = port 370 if headers: 371 self._tunnel_headers = headers 372 else: 373 self._tunnel_headers.clear() 374 375 def request(self, method, url, body=None, headers=None, *, 376 encode_chunked=False): 377 self.method = method 378 self.selector = url 379 if headers is not None: 380 self.req_headers += headers.items() 381 self.req_headers.sort() 382 if body: 383 self.data = body 384 self.encode_chunked = encode_chunked 385 if self.raise_on_endheaders: 386 raise OSError() 387 388 def getresponse(self): 389 return MockHTTPResponse(MockFile(), {}, 200, "OK") 390 391 def close(self): 392 pass 393 394 395class MockHandler: 396 # useful for testing handler machinery 397 # see add_ordered_mock_handlers() docstring 398 handler_order = 500 399 400 def __init__(self, methods): 401 self._define_methods(methods) 402 403 def _define_methods(self, methods): 404 for spec in methods: 405 if len(spec) == 2: 406 name, action = spec 407 else: 408 name, action = spec, None 409 meth = FakeMethod(name, action, self.handle) 410 setattr(self.__class__, name, meth) 411 412 def handle(self, fn_name, action, *args, **kwds): 413 self.parent.calls.append((self, fn_name, args, kwds)) 414 if action is None: 415 return None 416 elif action == "return self": 417 return self 418 elif action == "return response": 419 res = MockResponse(200, "OK", {}, "") 420 return res 421 elif action == "return request": 422 return Request("http://blah/") 423 elif action.startswith("error"): 424 code = action[action.rfind(" ")+1:] 425 try: 426 code = int(code) 427 except ValueError: 428 pass 429 res = MockResponse(200, "OK", {}, "") 430 return self.parent.error("http", args[0], res, code, "", {}) 431 elif action == "raise": 432 raise urllib.error.URLError("blah") 433 assert False 434 435 def close(self): 436 pass 437 438 def add_parent(self, parent): 439 self.parent = parent 440 self.parent.calls = [] 441 442 def __lt__(self, other): 443 if not hasattr(other, "handler_order"): 444 # No handler_order, leave in original order. Yuck. 445 return True 446 return self.handler_order < other.handler_order 447 448 449def add_ordered_mock_handlers(opener, meth_spec): 450 """Create MockHandlers and add them to an OpenerDirector. 451 452 meth_spec: list of lists of tuples and strings defining methods to define 453 on handlers. eg: 454 455 [["http_error", "ftp_open"], ["http_open"]] 456 457 defines methods .http_error() and .ftp_open() on one handler, and 458 .http_open() on another. These methods just record their arguments and 459 return None. Using a tuple instead of a string causes the method to 460 perform some action (see MockHandler.handle()), eg: 461 462 [["http_error"], [("http_open", "return request")]] 463 464 defines .http_error() on one handler (which simply returns None), and 465 .http_open() on another handler, which returns a Request object. 466 467 """ 468 handlers = [] 469 count = 0 470 for meths in meth_spec: 471 class MockHandlerSubclass(MockHandler): 472 pass 473 474 h = MockHandlerSubclass(meths) 475 h.handler_order += count 476 h.add_parent(opener) 477 count = count + 1 478 handlers.append(h) 479 opener.add_handler(h) 480 return handlers 481 482 483def build_test_opener(*handler_instances): 484 opener = OpenerDirector() 485 for h in handler_instances: 486 opener.add_handler(h) 487 return opener 488 489 490class MockHTTPHandler(urllib.request.HTTPHandler): 491 # Very simple mock HTTP handler with no special behavior other than using a mock HTTP connection 492 493 def __init__(self, debuglevel=None): 494 super(MockHTTPHandler, self).__init__(debuglevel=debuglevel) 495 self.httpconn = MockHTTPClass() 496 497 def http_open(self, req): 498 return self.do_open(self.httpconn, req) 499 500 501class MockHTTPHandlerRedirect(urllib.request.BaseHandler): 502 # useful for testing redirections and auth 503 # sends supplied headers and code as first response 504 # sends 200 OK as second response 505 def __init__(self, code, headers): 506 self.code = code 507 self.headers = headers 508 self.reset() 509 510 def reset(self): 511 self._count = 0 512 self.requests = [] 513 514 def http_open(self, req): 515 import email, copy 516 self.requests.append(copy.deepcopy(req)) 517 if self._count == 0: 518 self._count = self._count + 1 519 name = http.client.responses[self.code] 520 msg = email.message_from_string(self.headers) 521 return self.parent.error( 522 "http", req, MockFile(), self.code, name, msg) 523 else: 524 self.req = req 525 msg = email.message_from_string("\r\n\r\n") 526 return MockResponse(200, "OK", msg, "", req.get_full_url()) 527 528 529if hasattr(http.client, 'HTTPSConnection'): 530 class MockHTTPSHandler(urllib.request.HTTPSHandler): 531 # Useful for testing the Proxy-Authorization request by verifying the 532 # properties of httpcon 533 534 def __init__(self, debuglevel=None, context=None, check_hostname=None): 535 super(MockHTTPSHandler, self).__init__(debuglevel, context, check_hostname) 536 self.httpconn = MockHTTPClass() 537 538 def https_open(self, req): 539 return self.do_open(self.httpconn, req) 540 541 542class MockHTTPHandlerCheckAuth(urllib.request.BaseHandler): 543 # useful for testing auth 544 # sends supplied code response 545 # checks if auth header is specified in request 546 def __init__(self, code): 547 self.code = code 548 self.has_auth_header = False 549 550 def reset(self): 551 self.has_auth_header = False 552 553 def http_open(self, req): 554 if req.has_header('Authorization'): 555 self.has_auth_header = True 556 name = http.client.responses[self.code] 557 return MockResponse(self.code, name, MockFile(), "", req.get_full_url()) 558 559 560 561class MockPasswordManager: 562 def add_password(self, realm, uri, user, password): 563 self.realm = realm 564 self.url = uri 565 self.user = user 566 self.password = password 567 568 def find_user_password(self, realm, authuri): 569 self.target_realm = realm 570 self.target_url = authuri 571 return self.user, self.password 572 573 574class OpenerDirectorTests(unittest.TestCase): 575 576 def test_add_non_handler(self): 577 class NonHandler(object): 578 pass 579 self.assertRaises(TypeError, 580 OpenerDirector().add_handler, NonHandler()) 581 582 def test_badly_named_methods(self): 583 # test work-around for three methods that accidentally follow the 584 # naming conventions for handler methods 585 # (*_open() / *_request() / *_response()) 586 587 # These used to call the accidentally-named methods, causing a 588 # TypeError in real code; here, returning self from these mock 589 # methods would either cause no exception, or AttributeError. 590 591 from urllib.error import URLError 592 593 o = OpenerDirector() 594 meth_spec = [ 595 [("do_open", "return self"), ("proxy_open", "return self")], 596 [("redirect_request", "return self")], 597 ] 598 add_ordered_mock_handlers(o, meth_spec) 599 o.add_handler(urllib.request.UnknownHandler()) 600 for scheme in "do", "proxy", "redirect": 601 self.assertRaises(URLError, o.open, scheme+"://example.com/") 602 603 def test_handled(self): 604 # handler returning non-None means no more handlers will be called 605 o = OpenerDirector() 606 meth_spec = [ 607 ["http_open", "ftp_open", "http_error_302"], 608 ["ftp_open"], 609 [("http_open", "return self")], 610 [("http_open", "return self")], 611 ] 612 handlers = add_ordered_mock_handlers(o, meth_spec) 613 614 req = Request("http://example.com/") 615 r = o.open(req) 616 # Second .http_open() gets called, third doesn't, since second returned 617 # non-None. Handlers without .http_open() never get any methods called 618 # on them. 619 # In fact, second mock handler defining .http_open() returns self 620 # (instead of response), which becomes the OpenerDirector's return 621 # value. 622 self.assertEqual(r, handlers[2]) 623 calls = [(handlers[0], "http_open"), (handlers[2], "http_open")] 624 for expected, got in zip(calls, o.calls): 625 handler, name, args, kwds = got 626 self.assertEqual((handler, name), expected) 627 self.assertEqual(args, (req,)) 628 629 def test_handler_order(self): 630 o = OpenerDirector() 631 handlers = [] 632 for meths, handler_order in [([("http_open", "return self")], 500), 633 (["http_open"], 0)]: 634 class MockHandlerSubclass(MockHandler): 635 pass 636 637 h = MockHandlerSubclass(meths) 638 h.handler_order = handler_order 639 handlers.append(h) 640 o.add_handler(h) 641 642 o.open("http://example.com/") 643 # handlers called in reverse order, thanks to their sort order 644 self.assertEqual(o.calls[0][0], handlers[1]) 645 self.assertEqual(o.calls[1][0], handlers[0]) 646 647 def test_raise(self): 648 # raising URLError stops processing of request 649 o = OpenerDirector() 650 meth_spec = [ 651 [("http_open", "raise")], 652 [("http_open", "return self")], 653 ] 654 handlers = add_ordered_mock_handlers(o, meth_spec) 655 656 req = Request("http://example.com/") 657 self.assertRaises(urllib.error.URLError, o.open, req) 658 self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) 659 660 def test_http_error(self): 661 # XXX http_error_default 662 # http errors are a special case 663 o = OpenerDirector() 664 meth_spec = [ 665 [("http_open", "error 302")], 666 [("http_error_400", "raise"), "http_open"], 667 [("http_error_302", "return response"), "http_error_303", 668 "http_error"], 669 [("http_error_302")], 670 ] 671 handlers = add_ordered_mock_handlers(o, meth_spec) 672 req = Request("http://example.com/") 673 o.open(req) 674 assert len(o.calls) == 2 675 calls = [(handlers[0], "http_open", (req,)), 676 (handlers[2], "http_error_302", 677 (req, support.ALWAYS_EQ, 302, "", {}))] 678 for expected, got in zip(calls, o.calls): 679 handler, method_name, args = expected 680 self.assertEqual((handler, method_name), got[:2]) 681 self.assertEqual(args, got[2]) 682 683 def test_processors(self): 684 # *_request / *_response methods get called appropriately 685 o = OpenerDirector() 686 meth_spec = [ 687 [("http_request", "return request"), 688 ("http_response", "return response")], 689 [("http_request", "return request"), 690 ("http_response", "return response")], 691 ] 692 handlers = add_ordered_mock_handlers(o, meth_spec) 693 694 req = Request("http://example.com/") 695 o.open(req) 696 # processor methods are called on *all* handlers that define them, 697 # not just the first handler that handles the request 698 calls = [ 699 (handlers[0], "http_request"), (handlers[1], "http_request"), 700 (handlers[0], "http_response"), (handlers[1], "http_response")] 701 702 for i, (handler, name, args, kwds) in enumerate(o.calls): 703 if i < 2: 704 # *_request 705 self.assertEqual((handler, name), calls[i]) 706 self.assertEqual(len(args), 1) 707 self.assertIsInstance(args[0], Request) 708 else: 709 # *_response 710 self.assertEqual((handler, name), calls[i]) 711 self.assertEqual(len(args), 2) 712 self.assertIsInstance(args[0], Request) 713 # response from opener.open is None, because there's no 714 # handler that defines http_open to handle it 715 if args[1] is not None: 716 self.assertIsInstance(args[1], MockResponse) 717 718 719def sanepathname2url(path): 720 urlpath = urllib.request.pathname2url(path) 721 if os.name == "nt" and urlpath.startswith("///"): 722 urlpath = urlpath[2:] 723 # XXX don't ask me about the mac... 724 return urlpath 725 726 727class HandlerTests(unittest.TestCase): 728 729 def test_ftp(self): 730 class MockFTPWrapper: 731 def __init__(self, data): 732 self.data = data 733 734 def retrfile(self, filename, filetype): 735 self.filename, self.filetype = filename, filetype 736 return io.StringIO(self.data), len(self.data) 737 738 def close(self): 739 pass 740 741 class NullFTPHandler(urllib.request.FTPHandler): 742 def __init__(self, data): 743 self.data = data 744 745 def connect_ftp(self, user, passwd, host, port, dirs, 746 timeout=socket._GLOBAL_DEFAULT_TIMEOUT): 747 self.user, self.passwd = user, passwd 748 self.host, self.port = host, port 749 self.dirs = dirs 750 self.ftpwrapper = MockFTPWrapper(self.data) 751 return self.ftpwrapper 752 753 import ftplib 754 data = "rheum rhaponicum" 755 h = NullFTPHandler(data) 756 h.parent = MockOpener() 757 758 for url, host, port, user, passwd, type_, dirs, filename, mimetype in [ 759 ("ftp://localhost/foo/bar/baz.html", 760 "localhost", ftplib.FTP_PORT, "", "", "I", 761 ["foo", "bar"], "baz.html", "text/html"), 762 ("ftp://parrot@localhost/foo/bar/baz.html", 763 "localhost", ftplib.FTP_PORT, "parrot", "", "I", 764 ["foo", "bar"], "baz.html", "text/html"), 765 ("ftp://%25parrot@localhost/foo/bar/baz.html", 766 "localhost", ftplib.FTP_PORT, "%parrot", "", "I", 767 ["foo", "bar"], "baz.html", "text/html"), 768 ("ftp://%2542parrot@localhost/foo/bar/baz.html", 769 "localhost", ftplib.FTP_PORT, "%42parrot", "", "I", 770 ["foo", "bar"], "baz.html", "text/html"), 771 ("ftp://localhost:80/foo/bar/", 772 "localhost", 80, "", "", "D", 773 ["foo", "bar"], "", None), 774 ("ftp://localhost/baz.gif;type=a", 775 "localhost", ftplib.FTP_PORT, "", "", "A", 776 [], "baz.gif", "image/gif"), 777 ]: 778 req = Request(url) 779 req.timeout = None 780 r = h.ftp_open(req) 781 # ftp authentication not yet implemented by FTPHandler 782 self.assertEqual(h.user, user) 783 self.assertEqual(h.passwd, passwd) 784 self.assertEqual(h.host, socket.gethostbyname(host)) 785 self.assertEqual(h.port, port) 786 self.assertEqual(h.dirs, dirs) 787 self.assertEqual(h.ftpwrapper.filename, filename) 788 self.assertEqual(h.ftpwrapper.filetype, type_) 789 headers = r.info() 790 self.assertEqual(headers.get("Content-type"), mimetype) 791 self.assertEqual(int(headers["Content-length"]), len(data)) 792 793 def test_file(self): 794 import email.utils 795 h = urllib.request.FileHandler() 796 o = h.parent = MockOpener() 797 798 TESTFN = os_helper.TESTFN 799 urlpath = sanepathname2url(os.path.abspath(TESTFN)) 800 towrite = b"hello, world\n" 801 urls = [ 802 "file://localhost%s" % urlpath, 803 "file://%s" % urlpath, 804 "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), 805 ] 806 try: 807 localaddr = socket.gethostbyname(socket.gethostname()) 808 except socket.gaierror: 809 localaddr = '' 810 if localaddr: 811 urls.append("file://%s%s" % (localaddr, urlpath)) 812 813 for url in urls: 814 f = open(TESTFN, "wb") 815 try: 816 try: 817 f.write(towrite) 818 finally: 819 f.close() 820 821 r = h.file_open(Request(url)) 822 try: 823 data = r.read() 824 headers = r.info() 825 respurl = r.geturl() 826 finally: 827 r.close() 828 stats = os.stat(TESTFN) 829 modified = email.utils.formatdate(stats.st_mtime, usegmt=True) 830 finally: 831 os.remove(TESTFN) 832 self.assertEqual(data, towrite) 833 self.assertEqual(headers["Content-type"], "text/plain") 834 self.assertEqual(headers["Content-length"], "13") 835 self.assertEqual(headers["Last-modified"], modified) 836 self.assertEqual(respurl, url) 837 838 for url in [ 839 "file://localhost:80%s" % urlpath, 840 "file:///file_does_not_exist.txt", 841 "file://not-a-local-host.com//dir/file.txt", 842 "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), 843 os.getcwd(), TESTFN), 844 "file://somerandomhost.ontheinternet.com%s/%s" % 845 (os.getcwd(), TESTFN), 846 ]: 847 try: 848 f = open(TESTFN, "wb") 849 try: 850 f.write(towrite) 851 finally: 852 f.close() 853 854 self.assertRaises(urllib.error.URLError, 855 h.file_open, Request(url)) 856 finally: 857 os.remove(TESTFN) 858 859 h = urllib.request.FileHandler() 860 o = h.parent = MockOpener() 861 # XXXX why does // mean ftp (and /// mean not ftp!), and where 862 # is file: scheme specified? I think this is really a bug, and 863 # what was intended was to distinguish between URLs like: 864 # file:/blah.txt (a file) 865 # file://localhost/blah.txt (a file) 866 # file:///blah.txt (a file) 867 # file://ftp.example.com/blah.txt (an ftp URL) 868 for url, ftp in [ 869 ("file://ftp.example.com//foo.txt", False), 870 ("file://ftp.example.com///foo.txt", False), 871 ("file://ftp.example.com/foo.txt", False), 872 ("file://somehost//foo/something.txt", False), 873 ("file://localhost//foo/something.txt", False), 874 ]: 875 req = Request(url) 876 try: 877 h.file_open(req) 878 except urllib.error.URLError: 879 self.assertFalse(ftp) 880 else: 881 self.assertIs(o.req, req) 882 self.assertEqual(req.type, "ftp") 883 self.assertEqual(req.type == "ftp", ftp) 884 885 def test_http(self): 886 887 h = urllib.request.AbstractHTTPHandler() 888 o = h.parent = MockOpener() 889 890 url = "http://example.com/" 891 for method, data in [("GET", None), ("POST", b"blah")]: 892 req = Request(url, data, {"Foo": "bar"}) 893 req.timeout = None 894 req.add_unredirected_header("Spam", "eggs") 895 http = MockHTTPClass() 896 r = h.do_open(http, req) 897 898 # result attributes 899 r.read; r.readline # wrapped MockFile methods 900 r.info; r.geturl # addinfourl methods 901 r.code, r.msg == 200, "OK" # added from MockHTTPClass.getreply() 902 hdrs = r.info() 903 hdrs.get; hdrs.__contains__ # r.info() gives dict from .getreply() 904 self.assertEqual(r.geturl(), url) 905 906 self.assertEqual(http.host, "example.com") 907 self.assertEqual(http.level, 0) 908 self.assertEqual(http.method, method) 909 self.assertEqual(http.selector, "/") 910 self.assertEqual(http.req_headers, 911 [("Connection", "close"), 912 ("Foo", "bar"), ("Spam", "eggs")]) 913 self.assertEqual(http.data, data) 914 915 # check OSError converted to URLError 916 http.raise_on_endheaders = True 917 self.assertRaises(urllib.error.URLError, h.do_open, http, req) 918 919 # Check for TypeError on POST data which is str. 920 req = Request("http://example.com/","badpost") 921 self.assertRaises(TypeError, h.do_request_, req) 922 923 # check adding of standard headers 924 o.addheaders = [("Spam", "eggs")] 925 for data in b"", None: # POST, GET 926 req = Request("http://example.com/", data) 927 r = MockResponse(200, "OK", {}, "") 928 newreq = h.do_request_(req) 929 if data is None: # GET 930 self.assertNotIn("Content-length", req.unredirected_hdrs) 931 self.assertNotIn("Content-type", req.unredirected_hdrs) 932 else: # POST 933 self.assertEqual(req.unredirected_hdrs["Content-length"], "0") 934 self.assertEqual(req.unredirected_hdrs["Content-type"], 935 "application/x-www-form-urlencoded") 936 # XXX the details of Host could be better tested 937 self.assertEqual(req.unredirected_hdrs["Host"], "example.com") 938 self.assertEqual(req.unredirected_hdrs["Spam"], "eggs") 939 940 # don't clobber existing headers 941 req.add_unredirected_header("Content-length", "foo") 942 req.add_unredirected_header("Content-type", "bar") 943 req.add_unredirected_header("Host", "baz") 944 req.add_unredirected_header("Spam", "foo") 945 newreq = h.do_request_(req) 946 self.assertEqual(req.unredirected_hdrs["Content-length"], "foo") 947 self.assertEqual(req.unredirected_hdrs["Content-type"], "bar") 948 self.assertEqual(req.unredirected_hdrs["Host"], "baz") 949 self.assertEqual(req.unredirected_hdrs["Spam"], "foo") 950 951 def test_http_body_file(self): 952 # A regular file - chunked encoding is used unless Content Length is 953 # already set. 954 955 h = urllib.request.AbstractHTTPHandler() 956 o = h.parent = MockOpener() 957 958 file_obj = tempfile.NamedTemporaryFile(mode='w+b', delete=False) 959 file_path = file_obj.name 960 file_obj.close() 961 self.addCleanup(os.unlink, file_path) 962 963 with open(file_path, "rb") as f: 964 req = Request("http://example.com/", f, {}) 965 newreq = h.do_request_(req) 966 te = newreq.get_header('Transfer-encoding') 967 self.assertEqual(te, "chunked") 968 self.assertFalse(newreq.has_header('Content-length')) 969 970 with open(file_path, "rb") as f: 971 req = Request("http://example.com/", f, {"Content-Length": 30}) 972 newreq = h.do_request_(req) 973 self.assertEqual(int(newreq.get_header('Content-length')), 30) 974 self.assertFalse(newreq.has_header("Transfer-encoding")) 975 976 def test_http_body_fileobj(self): 977 # A file object - chunked encoding is used 978 # unless Content Length is already set. 979 # (Note that there are some subtle differences to a regular 980 # file, that is why we are testing both cases.) 981 982 h = urllib.request.AbstractHTTPHandler() 983 o = h.parent = MockOpener() 984 file_obj = io.BytesIO() 985 986 req = Request("http://example.com/", file_obj, {}) 987 newreq = h.do_request_(req) 988 self.assertEqual(newreq.get_header('Transfer-encoding'), 'chunked') 989 self.assertFalse(newreq.has_header('Content-length')) 990 991 headers = {"Content-Length": 30} 992 req = Request("http://example.com/", file_obj, headers) 993 newreq = h.do_request_(req) 994 self.assertEqual(int(newreq.get_header('Content-length')), 30) 995 self.assertFalse(newreq.has_header("Transfer-encoding")) 996 997 file_obj.close() 998 999 @requires_subprocess() 1000 def test_http_body_pipe(self): 1001 # A file reading from a pipe. 1002 # A pipe cannot be seek'ed. There is no way to determine the 1003 # content length up front. Thus, do_request_() should fall 1004 # back to Transfer-encoding chunked. 1005 1006 h = urllib.request.AbstractHTTPHandler() 1007 o = h.parent = MockOpener() 1008 1009 cmd = [sys.executable, "-c", r"pass"] 1010 for headers in {}, {"Content-Length": 30}: 1011 with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: 1012 req = Request("http://example.com/", proc.stdout, headers) 1013 newreq = h.do_request_(req) 1014 if not headers: 1015 self.assertEqual(newreq.get_header('Content-length'), None) 1016 self.assertEqual(newreq.get_header('Transfer-encoding'), 1017 'chunked') 1018 else: 1019 self.assertEqual(int(newreq.get_header('Content-length')), 1020 30) 1021 1022 def test_http_body_iterable(self): 1023 # Generic iterable. There is no way to determine the content 1024 # length up front. Fall back to Transfer-encoding chunked. 1025 1026 h = urllib.request.AbstractHTTPHandler() 1027 o = h.parent = MockOpener() 1028 1029 def iterable_body(): 1030 yield b"one" 1031 1032 for headers in {}, {"Content-Length": 11}: 1033 req = Request("http://example.com/", iterable_body(), headers) 1034 newreq = h.do_request_(req) 1035 if not headers: 1036 self.assertEqual(newreq.get_header('Content-length'), None) 1037 self.assertEqual(newreq.get_header('Transfer-encoding'), 1038 'chunked') 1039 else: 1040 self.assertEqual(int(newreq.get_header('Content-length')), 11) 1041 1042 def test_http_body_empty_seq(self): 1043 # Zero-length iterable body should be treated like any other iterable 1044 h = urllib.request.AbstractHTTPHandler() 1045 h.parent = MockOpener() 1046 req = h.do_request_(Request("http://example.com/", ())) 1047 self.assertEqual(req.get_header("Transfer-encoding"), "chunked") 1048 self.assertFalse(req.has_header("Content-length")) 1049 1050 def test_http_body_array(self): 1051 # array.array Iterable - Content Length is calculated 1052 1053 h = urllib.request.AbstractHTTPHandler() 1054 o = h.parent = MockOpener() 1055 1056 iterable_array = array.array("I",[1,2,3,4]) 1057 1058 for headers in {}, {"Content-Length": 16}: 1059 req = Request("http://example.com/", iterable_array, headers) 1060 newreq = h.do_request_(req) 1061 self.assertEqual(int(newreq.get_header('Content-length')),16) 1062 1063 def test_http_handler_global_debuglevel(self): 1064 with mock.patch.object(http.client.HTTPConnection, 'debuglevel', 6): 1065 o = OpenerDirector() 1066 h = MockHTTPHandler() 1067 o.add_handler(h) 1068 o.open("http://www.example.com") 1069 self.assertEqual(h._debuglevel, 6) 1070 1071 def test_http_handler_local_debuglevel(self): 1072 o = OpenerDirector() 1073 h = MockHTTPHandler(debuglevel=5) 1074 o.add_handler(h) 1075 o.open("http://www.example.com") 1076 self.assertEqual(h._debuglevel, 5) 1077 1078 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') 1079 def test_https_handler_global_debuglevel(self): 1080 with mock.patch.object(http.client.HTTPSConnection, 'debuglevel', 7): 1081 o = OpenerDirector() 1082 h = MockHTTPSHandler() 1083 o.add_handler(h) 1084 o.open("https://www.example.com") 1085 self.assertEqual(h._debuglevel, 7) 1086 1087 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') 1088 def test_https_handler_local_debuglevel(self): 1089 o = OpenerDirector() 1090 h = MockHTTPSHandler(debuglevel=4) 1091 o.add_handler(h) 1092 o.open("https://www.example.com") 1093 self.assertEqual(h._debuglevel, 4) 1094 1095 def test_http_doubleslash(self): 1096 # Checks the presence of any unnecessary double slash in url does not 1097 # break anything. Previously, a double slash directly after the host 1098 # could cause incorrect parsing. 1099 h = urllib.request.AbstractHTTPHandler() 1100 h.parent = MockOpener() 1101 1102 data = b"" 1103 ds_urls = [ 1104 "http://example.com/foo/bar/baz.html", 1105 "http://example.com//foo/bar/baz.html", 1106 "http://example.com/foo//bar/baz.html", 1107 "http://example.com/foo/bar//baz.html" 1108 ] 1109 1110 for ds_url in ds_urls: 1111 ds_req = Request(ds_url, data) 1112 1113 # Check whether host is determined correctly if there is no proxy 1114 np_ds_req = h.do_request_(ds_req) 1115 self.assertEqual(np_ds_req.unredirected_hdrs["Host"], "example.com") 1116 1117 # Check whether host is determined correctly if there is a proxy 1118 ds_req.set_proxy("someproxy:3128", None) 1119 p_ds_req = h.do_request_(ds_req) 1120 self.assertEqual(p_ds_req.unredirected_hdrs["Host"], "example.com") 1121 1122 def test_full_url_setter(self): 1123 # Checks to ensure that components are set correctly after setting the 1124 # full_url of a Request object 1125 1126 urls = [ 1127 'http://example.com?foo=bar#baz', 1128 'http://example.com?foo=bar&spam=eggs#bash', 1129 'http://example.com', 1130 ] 1131 1132 # testing a reusable request instance, but the url parameter is 1133 # required, so just use a dummy one to instantiate 1134 r = Request('http://example.com') 1135 for url in urls: 1136 r.full_url = url 1137 parsed = urlparse(url) 1138 1139 self.assertEqual(r.get_full_url(), url) 1140 # full_url setter uses splittag to split into components. 1141 # splittag sets the fragment as None while urlparse sets it to '' 1142 self.assertEqual(r.fragment or '', parsed.fragment) 1143 self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) 1144 1145 def test_full_url_deleter(self): 1146 r = Request('http://www.example.com') 1147 del r.full_url 1148 self.assertIsNone(r.full_url) 1149 self.assertIsNone(r.fragment) 1150 self.assertEqual(r.selector, '') 1151 1152 def test_fixpath_in_weirdurls(self): 1153 # Issue4493: urllib2 to supply '/' when to urls where path does not 1154 # start with'/' 1155 1156 h = urllib.request.AbstractHTTPHandler() 1157 h.parent = MockOpener() 1158 1159 weird_url = 'http://www.python.org?getspam' 1160 req = Request(weird_url) 1161 newreq = h.do_request_(req) 1162 self.assertEqual(newreq.host, 'www.python.org') 1163 self.assertEqual(newreq.selector, '/?getspam') 1164 1165 url_without_path = 'http://www.python.org' 1166 req = Request(url_without_path) 1167 newreq = h.do_request_(req) 1168 self.assertEqual(newreq.host, 'www.python.org') 1169 self.assertEqual(newreq.selector, '') 1170 1171 def test_errors(self): 1172 h = urllib.request.HTTPErrorProcessor() 1173 o = h.parent = MockOpener() 1174 1175 url = "http://example.com/" 1176 req = Request(url) 1177 # all 2xx are passed through 1178 r = MockResponse(200, "OK", {}, "", url) 1179 newr = h.http_response(req, r) 1180 self.assertIs(r, newr) 1181 self.assertFalse(hasattr(o, "proto")) # o.error not called 1182 r = MockResponse(202, "Accepted", {}, "", url) 1183 newr = h.http_response(req, r) 1184 self.assertIs(r, newr) 1185 self.assertFalse(hasattr(o, "proto")) # o.error not called 1186 r = MockResponse(206, "Partial content", {}, "", url) 1187 newr = h.http_response(req, r) 1188 self.assertIs(r, newr) 1189 self.assertFalse(hasattr(o, "proto")) # o.error not called 1190 # anything else calls o.error (and MockOpener returns None, here) 1191 r = MockResponse(502, "Bad gateway", {}, "", url) 1192 self.assertIsNone(h.http_response(req, r)) 1193 self.assertEqual(o.proto, "http") # o.error called 1194 self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) 1195 1196 def test_cookies(self): 1197 cj = MockCookieJar() 1198 h = urllib.request.HTTPCookieProcessor(cj) 1199 h.parent = MockOpener() 1200 1201 req = Request("http://example.com/") 1202 r = MockResponse(200, "OK", {}, "") 1203 newreq = h.http_request(req) 1204 self.assertIs(cj.ach_req, req) 1205 self.assertIs(cj.ach_req, newreq) 1206 self.assertEqual(req.origin_req_host, "example.com") 1207 self.assertFalse(req.unverifiable) 1208 newr = h.http_response(req, r) 1209 self.assertIs(cj.ec_req, req) 1210 self.assertIs(cj.ec_r, r) 1211 self.assertIs(r, newr) 1212 1213 def test_redirect(self): 1214 from_url = "http://example.com/a.html" 1215 to_url = "http://example.com/b.html" 1216 h = urllib.request.HTTPRedirectHandler() 1217 o = h.parent = MockOpener() 1218 1219 # ordinary redirect behaviour 1220 for code in 301, 302, 303, 307, 308: 1221 for data in None, "blah\nblah\n": 1222 method = getattr(h, "http_error_%s" % code) 1223 req = Request(from_url, data) 1224 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1225 req.add_header("Nonsense", "viking=withhold") 1226 if data is not None: 1227 req.add_header("Content-Length", str(len(data))) 1228 req.add_unredirected_header("Spam", "spam") 1229 try: 1230 method(req, MockFile(), code, "Blah", 1231 MockHeaders({"location": to_url})) 1232 except urllib.error.HTTPError: 1233 # 307 and 308 in response to POST require user OK 1234 self.assertIn(code, (307, 308)) 1235 self.assertIsNotNone(data) 1236 self.assertEqual(o.req.get_full_url(), to_url) 1237 try: 1238 self.assertEqual(o.req.get_method(), "GET") 1239 except AttributeError: 1240 self.assertFalse(o.req.data) 1241 1242 # now it's a GET, there should not be headers regarding content 1243 # (possibly dragged from before being a POST) 1244 headers = [x.lower() for x in o.req.headers] 1245 self.assertNotIn("content-length", headers) 1246 self.assertNotIn("content-type", headers) 1247 1248 self.assertEqual(o.req.headers["Nonsense"], 1249 "viking=withhold") 1250 self.assertNotIn("Spam", o.req.headers) 1251 self.assertNotIn("Spam", o.req.unredirected_hdrs) 1252 1253 # loop detection 1254 req = Request(from_url) 1255 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1256 1257 def redirect(h, req, url=to_url): 1258 h.http_error_302(req, MockFile(), 302, "Blah", 1259 MockHeaders({"location": url})) 1260 # Note that the *original* request shares the same record of 1261 # redirections with the sub-requests caused by the redirections. 1262 1263 # detect infinite loop redirect of a URL to itself 1264 req = Request(from_url, origin_req_host="example.com") 1265 count = 0 1266 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1267 try: 1268 while 1: 1269 redirect(h, req, "http://example.com/") 1270 count = count + 1 1271 except urllib.error.HTTPError: 1272 # don't stop until max_repeats, because cookies may introduce state 1273 self.assertEqual(count, urllib.request.HTTPRedirectHandler.max_repeats) 1274 1275 # detect endless non-repeating chain of redirects 1276 req = Request(from_url, origin_req_host="example.com") 1277 count = 0 1278 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1279 try: 1280 while 1: 1281 redirect(h, req, "http://example.com/%d" % count) 1282 count = count + 1 1283 except urllib.error.HTTPError: 1284 self.assertEqual(count, 1285 urllib.request.HTTPRedirectHandler.max_redirections) 1286 1287 def test_invalid_redirect(self): 1288 from_url = "http://example.com/a.html" 1289 valid_schemes = ['http','https','ftp'] 1290 invalid_schemes = ['file','imap','ldap'] 1291 schemeless_url = "example.com/b.html" 1292 h = urllib.request.HTTPRedirectHandler() 1293 o = h.parent = MockOpener() 1294 req = Request(from_url) 1295 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1296 1297 for scheme in invalid_schemes: 1298 invalid_url = scheme + '://' + schemeless_url 1299 self.assertRaises(urllib.error.HTTPError, h.http_error_302, 1300 req, MockFile(), 302, "Security Loophole", 1301 MockHeaders({"location": invalid_url})) 1302 1303 for scheme in valid_schemes: 1304 valid_url = scheme + '://' + schemeless_url 1305 h.http_error_302(req, MockFile(), 302, "That's fine", 1306 MockHeaders({"location": valid_url})) 1307 self.assertEqual(o.req.get_full_url(), valid_url) 1308 1309 def test_relative_redirect(self): 1310 from_url = "http://example.com/a.html" 1311 relative_url = "/b.html" 1312 h = urllib.request.HTTPRedirectHandler() 1313 o = h.parent = MockOpener() 1314 req = Request(from_url) 1315 req.timeout = socket._GLOBAL_DEFAULT_TIMEOUT 1316 1317 valid_url = urllib.parse.urljoin(from_url,relative_url) 1318 h.http_error_302(req, MockFile(), 302, "That's fine", 1319 MockHeaders({"location": valid_url})) 1320 self.assertEqual(o.req.get_full_url(), valid_url) 1321 1322 def test_cookie_redirect(self): 1323 # cookies shouldn't leak into redirected requests 1324 from http.cookiejar import CookieJar 1325 from test.test_http_cookiejar import interact_netscape 1326 1327 cj = CookieJar() 1328 interact_netscape(cj, "http://www.example.com/", "spam=eggs") 1329 hh = MockHTTPHandlerRedirect(302, "Location: http://www.cracker.com/\r\n\r\n") 1330 hdeh = urllib.request.HTTPDefaultErrorHandler() 1331 hrh = urllib.request.HTTPRedirectHandler() 1332 cp = urllib.request.HTTPCookieProcessor(cj) 1333 o = build_test_opener(hh, hdeh, hrh, cp) 1334 o.open("http://www.example.com/") 1335 self.assertFalse(hh.req.has_header("Cookie")) 1336 1337 def test_redirect_fragment(self): 1338 redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' 1339 hh = MockHTTPHandlerRedirect(302, 'Location: ' + redirected_url) 1340 hdeh = urllib.request.HTTPDefaultErrorHandler() 1341 hrh = urllib.request.HTTPRedirectHandler() 1342 o = build_test_opener(hh, hdeh, hrh) 1343 fp = o.open('http://www.example.com') 1344 self.assertEqual(fp.geturl(), redirected_url.strip()) 1345 1346 def test_redirect_no_path(self): 1347 # Issue 14132: Relative redirect strips original path 1348 1349 # clear _opener global variable 1350 self.addCleanup(urllib.request.urlcleanup) 1351 1352 real_class = http.client.HTTPConnection 1353 response1 = b"HTTP/1.1 302 Found\r\nLocation: ?query\r\n\r\n" 1354 http.client.HTTPConnection = test_urllib.fakehttp(response1) 1355 self.addCleanup(setattr, http.client, "HTTPConnection", real_class) 1356 urls = iter(("/path", "/path?query")) 1357 def request(conn, method, url, *pos, **kw): 1358 self.assertEqual(url, next(urls)) 1359 real_class.request(conn, method, url, *pos, **kw) 1360 # Change response for subsequent connection 1361 conn.__class__.fakedata = b"HTTP/1.1 200 OK\r\n\r\nHello!" 1362 http.client.HTTPConnection.request = request 1363 fp = urllib.request.urlopen("http://python.org/path") 1364 self.assertEqual(fp.geturl(), "http://python.org/path?query") 1365 1366 def test_redirect_encoding(self): 1367 # Some characters in the redirect target may need special handling, 1368 # but most ASCII characters should be treated as already encoded 1369 class Handler(urllib.request.HTTPHandler): 1370 def http_open(self, req): 1371 result = self.do_open(self.connection, req) 1372 self.last_buf = self.connection.buf 1373 # Set up a normal response for the next request 1374 self.connection = test_urllib.fakehttp( 1375 b'HTTP/1.1 200 OK\r\n' 1376 b'Content-Length: 3\r\n' 1377 b'\r\n' 1378 b'123' 1379 ) 1380 return result 1381 handler = Handler() 1382 opener = urllib.request.build_opener(handler) 1383 tests = ( 1384 (b'/p\xC3\xA5-dansk/', b'/p%C3%A5-dansk/'), 1385 (b'/spaced%20path/', b'/spaced%20path/'), 1386 (b'/spaced path/', b'/spaced%20path/'), 1387 (b'/?p\xC3\xA5-dansk', b'/?p%C3%A5-dansk'), 1388 ) 1389 for [location, result] in tests: 1390 with self.subTest(repr(location)): 1391 handler.connection = test_urllib.fakehttp( 1392 b'HTTP/1.1 302 Redirect\r\n' 1393 b'Location: ' + location + b'\r\n' 1394 b'\r\n' 1395 ) 1396 response = opener.open('http://example.com/') 1397 expected = b'GET ' + result + b' ' 1398 request = handler.last_buf 1399 self.assertTrue(request.startswith(expected), repr(request)) 1400 1401 def test_redirect_head_request(self): 1402 from_url = "http://example.com/a.html" 1403 to_url = "http://example.com/b.html" 1404 h = urllib.request.HTTPRedirectHandler() 1405 req = Request(from_url, method="HEAD") 1406 fp = MockFile() 1407 new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) 1408 self.assertEqual(new_req.get_method(), "HEAD") 1409 1410 def test_proxy(self): 1411 u = "proxy.example.com:3128" 1412 for d in dict(http=u), dict(HTTP=u): 1413 o = OpenerDirector() 1414 ph = urllib.request.ProxyHandler(d) 1415 o.add_handler(ph) 1416 meth_spec = [ 1417 [("http_open", "return response")] 1418 ] 1419 handlers = add_ordered_mock_handlers(o, meth_spec) 1420 1421 req = Request("http://acme.example.com/") 1422 self.assertEqual(req.host, "acme.example.com") 1423 o.open(req) 1424 self.assertEqual(req.host, u) 1425 self.assertEqual([(handlers[0], "http_open")], 1426 [tup[0:2] for tup in o.calls]) 1427 1428 def test_proxy_no_proxy(self): 1429 os.environ['no_proxy'] = 'python.org' 1430 o = OpenerDirector() 1431 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1432 o.add_handler(ph) 1433 req = Request("http://www.perl.org/") 1434 self.assertEqual(req.host, "www.perl.org") 1435 o.open(req) 1436 self.assertEqual(req.host, "proxy.example.com") 1437 req = Request("http://www.python.org") 1438 self.assertEqual(req.host, "www.python.org") 1439 o.open(req) 1440 self.assertEqual(req.host, "www.python.org") 1441 del os.environ['no_proxy'] 1442 1443 def test_proxy_no_proxy_all(self): 1444 os.environ['no_proxy'] = '*' 1445 o = OpenerDirector() 1446 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com")) 1447 o.add_handler(ph) 1448 req = Request("http://www.python.org") 1449 self.assertEqual(req.host, "www.python.org") 1450 o.open(req) 1451 self.assertEqual(req.host, "www.python.org") 1452 del os.environ['no_proxy'] 1453 1454 def test_proxy_https(self): 1455 o = OpenerDirector() 1456 ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) 1457 o.add_handler(ph) 1458 meth_spec = [ 1459 [("https_open", "return response")] 1460 ] 1461 handlers = add_ordered_mock_handlers(o, meth_spec) 1462 1463 req = Request("https://www.example.com/") 1464 self.assertEqual(req.host, "www.example.com") 1465 o.open(req) 1466 self.assertEqual(req.host, "proxy.example.com:3128") 1467 self.assertEqual([(handlers[0], "https_open")], 1468 [tup[0:2] for tup in o.calls]) 1469 1470 @unittest.skipUnless(hasattr(http.client, 'HTTPSConnection'), 'HTTPSConnection required for HTTPS tests.') 1471 def test_proxy_https_proxy_authorization(self): 1472 o = OpenerDirector() 1473 ph = urllib.request.ProxyHandler(dict(https='proxy.example.com:3128')) 1474 o.add_handler(ph) 1475 https_handler = MockHTTPSHandler() 1476 o.add_handler(https_handler) 1477 req = Request("https://www.example.com/") 1478 req.add_header("Proxy-Authorization", "FooBar") 1479 req.add_header("User-Agent", "Grail") 1480 self.assertEqual(req.host, "www.example.com") 1481 self.assertIsNone(req._tunnel_host) 1482 o.open(req) 1483 # Verify Proxy-Authorization gets tunneled to request. 1484 # httpsconn req_headers do not have the Proxy-Authorization header but 1485 # the req will have. 1486 self.assertNotIn(("Proxy-Authorization", "FooBar"), 1487 https_handler.httpconn.req_headers) 1488 self.assertIn(("User-Agent", "Grail"), 1489 https_handler.httpconn.req_headers) 1490 self.assertIsNotNone(req._tunnel_host) 1491 self.assertEqual(req.host, "proxy.example.com:3128") 1492 self.assertEqual(req.get_header("Proxy-authorization"), "FooBar") 1493 1494 @unittest.skipUnless(os.name == "nt", "only relevant for Windows") 1495 def test_winreg_proxy_bypass(self): 1496 proxy_override = "www.example.com;*.example.net; 192.168.0.1" 1497 proxy_bypass = _proxy_bypass_winreg_override 1498 for host in ("www.example.com", "www.example.net", "192.168.0.1"): 1499 self.assertTrue(proxy_bypass(host, proxy_override), 1500 "expected bypass of %s to be true" % host) 1501 1502 for host in ("example.com", "www.example.org", "example.net", 1503 "192.168.0.2"): 1504 self.assertFalse(proxy_bypass(host, proxy_override), 1505 "expected bypass of %s to be False" % host) 1506 1507 # check intranet address bypass 1508 proxy_override = "example.com; <local>" 1509 self.assertTrue(proxy_bypass("example.com", proxy_override), 1510 "expected bypass of %s to be true" % host) 1511 self.assertFalse(proxy_bypass("example.net", proxy_override), 1512 "expected bypass of %s to be False" % host) 1513 for host in ("test", "localhost"): 1514 self.assertTrue(proxy_bypass(host, proxy_override), 1515 "expect <local> to bypass intranet address '%s'" 1516 % host) 1517 1518 @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") 1519 def test_osx_proxy_bypass(self): 1520 bypass = { 1521 'exclude_simple': False, 1522 'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10', 1523 '10.0/16'] 1524 } 1525 # Check hosts that should trigger the proxy bypass 1526 for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1', 1527 '10.0.0.1'): 1528 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), 1529 'expected bypass of %s to be True' % host) 1530 # Check hosts that should not trigger the proxy bypass 1531 for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', 1532 'notinbypass'): 1533 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), 1534 'expected bypass of %s to be False' % host) 1535 1536 # Check the exclude_simple flag 1537 bypass = {'exclude_simple': True, 'exceptions': []} 1538 self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass)) 1539 1540 # Check that invalid prefix lengths are ignored 1541 bypass = { 1542 'exclude_simple': False, 1543 'exceptions': [ '10.0.0.0/40', '172.19.10.0/24' ] 1544 } 1545 host = '172.19.10.5' 1546 self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass), 1547 'expected bypass of %s to be True' % host) 1548 host = '10.0.1.5' 1549 self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass), 1550 'expected bypass of %s to be False' % host) 1551 1552 def check_basic_auth(self, headers, realm): 1553 with self.subTest(realm=realm, headers=headers): 1554 opener = OpenerDirector() 1555 password_manager = MockPasswordManager() 1556 auth_handler = urllib.request.HTTPBasicAuthHandler(password_manager) 1557 body = '\r\n'.join(headers) + '\r\n\r\n' 1558 http_handler = MockHTTPHandlerRedirect(401, body) 1559 opener.add_handler(auth_handler) 1560 opener.add_handler(http_handler) 1561 self._test_basic_auth(opener, auth_handler, "Authorization", 1562 realm, http_handler, password_manager, 1563 "http://acme.example.com/protected", 1564 "http://acme.example.com/protected") 1565 1566 def test_basic_auth(self): 1567 realm = "realm2@example.com" 1568 realm2 = "realm2@example.com" 1569 basic = f'Basic realm="{realm}"' 1570 basic2 = f'Basic realm="{realm2}"' 1571 other_no_realm = 'Otherscheme xxx' 1572 digest = (f'Digest realm="{realm2}", ' 1573 f'qop="auth, auth-int", ' 1574 f'nonce="dcd98b7102dd2f0e8b11d0f600bfb0c093", ' 1575 f'opaque="5ccc069c403ebaf9f0171e9517f40e41"') 1576 for realm_str in ( 1577 # test "quote" and 'quote' 1578 f'Basic realm="{realm}"', 1579 f"Basic realm='{realm}'", 1580 1581 # charset is ignored 1582 f'Basic realm="{realm}", charset="UTF-8"', 1583 1584 # Multiple challenges per header 1585 f'{basic}, {basic2}', 1586 f'{basic}, {other_no_realm}', 1587 f'{other_no_realm}, {basic}', 1588 f'{basic}, {digest}', 1589 f'{digest}, {basic}', 1590 ): 1591 headers = [f'WWW-Authenticate: {realm_str}'] 1592 self.check_basic_auth(headers, realm) 1593 1594 # no quote: expect a warning 1595 with warnings_helper.check_warnings(("Basic Auth Realm was unquoted", 1596 UserWarning)): 1597 headers = [f'WWW-Authenticate: Basic realm={realm}'] 1598 self.check_basic_auth(headers, realm) 1599 1600 # Multiple headers: one challenge per header. 1601 # Use the first Basic realm. 1602 for challenges in ( 1603 [basic, basic2], 1604 [basic, digest], 1605 [digest, basic], 1606 ): 1607 headers = [f'WWW-Authenticate: {challenge}' 1608 for challenge in challenges] 1609 self.check_basic_auth(headers, realm) 1610 1611 def test_proxy_basic_auth(self): 1612 opener = OpenerDirector() 1613 ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) 1614 opener.add_handler(ph) 1615 password_manager = MockPasswordManager() 1616 auth_handler = urllib.request.ProxyBasicAuthHandler(password_manager) 1617 realm = "ACME Networks" 1618 http_handler = MockHTTPHandlerRedirect( 1619 407, 'Proxy-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1620 opener.add_handler(auth_handler) 1621 opener.add_handler(http_handler) 1622 self._test_basic_auth(opener, auth_handler, "Proxy-authorization", 1623 realm, http_handler, password_manager, 1624 "http://acme.example.com:3128/protected", 1625 "proxy.example.com:3128", 1626 ) 1627 1628 def test_basic_and_digest_auth_handlers(self): 1629 # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* 1630 # response (https://bugs.python.org/issue1479302), where it should instead 1631 # return None to allow another handler (especially 1632 # HTTPBasicAuthHandler) to handle the response. 1633 1634 # Also (https://bugs.python.org/issue14797027, RFC 2617 section 1.2), we must 1635 # try digest first (since it's the strongest auth scheme), so we record 1636 # order of calls here to check digest comes first: 1637 class RecordingOpenerDirector(OpenerDirector): 1638 def __init__(self): 1639 OpenerDirector.__init__(self) 1640 self.recorded = [] 1641 1642 def record(self, info): 1643 self.recorded.append(info) 1644 1645 class TestDigestAuthHandler(urllib.request.HTTPDigestAuthHandler): 1646 def http_error_401(self, *args, **kwds): 1647 self.parent.record("digest") 1648 urllib.request.HTTPDigestAuthHandler.http_error_401(self, 1649 *args, **kwds) 1650 1651 class TestBasicAuthHandler(urllib.request.HTTPBasicAuthHandler): 1652 def http_error_401(self, *args, **kwds): 1653 self.parent.record("basic") 1654 urllib.request.HTTPBasicAuthHandler.http_error_401(self, 1655 *args, **kwds) 1656 1657 opener = RecordingOpenerDirector() 1658 password_manager = MockPasswordManager() 1659 digest_handler = TestDigestAuthHandler(password_manager) 1660 basic_handler = TestBasicAuthHandler(password_manager) 1661 realm = "ACME Networks" 1662 http_handler = MockHTTPHandlerRedirect( 1663 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % realm) 1664 opener.add_handler(basic_handler) 1665 opener.add_handler(digest_handler) 1666 opener.add_handler(http_handler) 1667 1668 # check basic auth isn't blocked by digest handler failing 1669 self._test_basic_auth(opener, basic_handler, "Authorization", 1670 realm, http_handler, password_manager, 1671 "http://acme.example.com/protected", 1672 "http://acme.example.com/protected", 1673 ) 1674 # check digest was tried before basic (twice, because 1675 # _test_basic_auth called .open() twice) 1676 self.assertEqual(opener.recorded, ["digest", "basic"]*2) 1677 1678 def test_unsupported_auth_digest_handler(self): 1679 opener = OpenerDirector() 1680 # While using DigestAuthHandler 1681 digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) 1682 http_handler = MockHTTPHandlerRedirect( 1683 401, 'WWW-Authenticate: Kerberos\r\n\r\n') 1684 opener.add_handler(digest_auth_handler) 1685 opener.add_handler(http_handler) 1686 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1687 1688 def test_unsupported_auth_basic_handler(self): 1689 # While using BasicAuthHandler 1690 opener = OpenerDirector() 1691 basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) 1692 http_handler = MockHTTPHandlerRedirect( 1693 401, 'WWW-Authenticate: NTLM\r\n\r\n') 1694 opener.add_handler(basic_auth_handler) 1695 opener.add_handler(http_handler) 1696 self.assertRaises(ValueError, opener.open, "http://www.example.com") 1697 1698 def _test_basic_auth(self, opener, auth_handler, auth_header, 1699 realm, http_handler, password_manager, 1700 request_url, protected_url): 1701 import base64 1702 user, password = "wile", "coyote" 1703 1704 # .add_password() fed through to password manager 1705 auth_handler.add_password(realm, request_url, user, password) 1706 self.assertEqual(realm, password_manager.realm) 1707 self.assertEqual(request_url, password_manager.url) 1708 self.assertEqual(user, password_manager.user) 1709 self.assertEqual(password, password_manager.password) 1710 1711 opener.open(request_url) 1712 1713 # should have asked the password manager for the username/password 1714 self.assertEqual(password_manager.target_realm, realm) 1715 self.assertEqual(password_manager.target_url, protected_url) 1716 1717 # expect one request without authorization, then one with 1718 self.assertEqual(len(http_handler.requests), 2) 1719 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1720 userpass = bytes('%s:%s' % (user, password), "ascii") 1721 auth_hdr_value = ('Basic ' + 1722 base64.encodebytes(userpass).strip().decode()) 1723 self.assertEqual(http_handler.requests[1].get_header(auth_header), 1724 auth_hdr_value) 1725 self.assertEqual(http_handler.requests[1].unredirected_hdrs[auth_header], 1726 auth_hdr_value) 1727 # if the password manager can't find a password, the handler won't 1728 # handle the HTTP auth error 1729 password_manager.user = password_manager.password = None 1730 http_handler.reset() 1731 opener.open(request_url) 1732 self.assertEqual(len(http_handler.requests), 1) 1733 self.assertFalse(http_handler.requests[0].has_header(auth_header)) 1734 1735 def test_basic_prior_auth_auto_send(self): 1736 # Assume already authenticated if is_authenticated=True 1737 # for APIs like Github that don't return 401 1738 1739 user, password = "wile", "coyote" 1740 request_url = "http://acme.example.com/protected" 1741 1742 http_handler = MockHTTPHandlerCheckAuth(200) 1743 1744 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1745 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1746 auth_prior_handler.add_password( 1747 None, request_url, user, password, is_authenticated=True) 1748 1749 self.assertTrue(pwd_manager.is_authenticated(request_url)) 1750 self.assertTrue(pwd_manager.is_authenticated(request_url + '/nested')) 1751 self.assertFalse(pwd_manager.is_authenticated(request_url + 'plain')) 1752 1753 opener = OpenerDirector() 1754 opener.add_handler(auth_prior_handler) 1755 opener.add_handler(http_handler) 1756 1757 opener.open(request_url) 1758 1759 # expect request to be sent with auth header 1760 self.assertTrue(http_handler.has_auth_header) 1761 1762 def test_basic_prior_auth_send_after_first_success(self): 1763 # Auto send auth header after authentication is successful once 1764 1765 user, password = 'wile', 'coyote' 1766 request_url = 'http://acme.example.com/protected' 1767 realm = 'ACME' 1768 1769 pwd_manager = HTTPPasswordMgrWithPriorAuth() 1770 auth_prior_handler = HTTPBasicAuthHandler(pwd_manager) 1771 auth_prior_handler.add_password(realm, request_url, user, password) 1772 1773 is_auth = pwd_manager.is_authenticated(request_url) 1774 self.assertFalse(is_auth) 1775 1776 opener = OpenerDirector() 1777 opener.add_handler(auth_prior_handler) 1778 1779 http_handler = MockHTTPHandlerRedirect( 1780 401, 'WWW-Authenticate: Basic realm="%s"\r\n\r\n' % None) 1781 opener.add_handler(http_handler) 1782 1783 opener.open(request_url) 1784 1785 is_auth = pwd_manager.is_authenticated(request_url) 1786 self.assertTrue(is_auth) 1787 1788 http_handler = MockHTTPHandlerCheckAuth(200) 1789 self.assertFalse(http_handler.has_auth_header) 1790 1791 opener = OpenerDirector() 1792 opener.add_handler(auth_prior_handler) 1793 opener.add_handler(http_handler) 1794 1795 # After getting 200 from MockHTTPHandler 1796 # Next request sends header in the first request 1797 opener.open(request_url) 1798 1799 # expect request to be sent with auth header 1800 self.assertTrue(http_handler.has_auth_header) 1801 1802 def test_http_closed(self): 1803 """Test the connection is cleaned up when the response is closed""" 1804 for (transfer, data) in ( 1805 ("Connection: close", b"data"), 1806 ("Transfer-Encoding: chunked", b"4\r\ndata\r\n0\r\n\r\n"), 1807 ("Content-Length: 4", b"data"), 1808 ): 1809 header = "HTTP/1.1 200 OK\r\n{}\r\n\r\n".format(transfer) 1810 conn = test_urllib.fakehttp(header.encode() + data) 1811 handler = urllib.request.AbstractHTTPHandler() 1812 req = Request("http://dummy/") 1813 req.timeout = None 1814 with handler.do_open(conn, req) as resp: 1815 resp.read() 1816 self.assertTrue(conn.fakesock.closed, 1817 "Connection not closed with {!r}".format(transfer)) 1818 1819 def test_invalid_closed(self): 1820 """Test the connection is cleaned up after an invalid response""" 1821 conn = test_urllib.fakehttp(b"") 1822 handler = urllib.request.AbstractHTTPHandler() 1823 req = Request("http://dummy/") 1824 req.timeout = None 1825 with self.assertRaises(http.client.BadStatusLine): 1826 handler.do_open(conn, req) 1827 self.assertTrue(conn.fakesock.closed, "Connection not closed") 1828 1829 1830class MiscTests(unittest.TestCase): 1831 1832 def opener_has_handler(self, opener, handler_class): 1833 self.assertTrue(any(h.__class__ == handler_class 1834 for h in opener.handlers)) 1835 1836 def test_build_opener(self): 1837 class MyHTTPHandler(urllib.request.HTTPHandler): 1838 pass 1839 1840 class FooHandler(urllib.request.BaseHandler): 1841 def foo_open(self): 1842 pass 1843 1844 class BarHandler(urllib.request.BaseHandler): 1845 def bar_open(self): 1846 pass 1847 1848 build_opener = urllib.request.build_opener 1849 1850 o = build_opener(FooHandler, BarHandler) 1851 self.opener_has_handler(o, FooHandler) 1852 self.opener_has_handler(o, BarHandler) 1853 1854 # can take a mix of classes and instances 1855 o = build_opener(FooHandler, BarHandler()) 1856 self.opener_has_handler(o, FooHandler) 1857 self.opener_has_handler(o, BarHandler) 1858 1859 # subclasses of default handlers override default handlers 1860 o = build_opener(MyHTTPHandler) 1861 self.opener_has_handler(o, MyHTTPHandler) 1862 1863 # a particular case of overriding: default handlers can be passed 1864 # in explicitly 1865 o = build_opener() 1866 self.opener_has_handler(o, urllib.request.HTTPHandler) 1867 o = build_opener(urllib.request.HTTPHandler) 1868 self.opener_has_handler(o, urllib.request.HTTPHandler) 1869 o = build_opener(urllib.request.HTTPHandler()) 1870 self.opener_has_handler(o, urllib.request.HTTPHandler) 1871 1872 # Issue2670: multiple handlers sharing the same base class 1873 class MyOtherHTTPHandler(urllib.request.HTTPHandler): 1874 pass 1875 1876 o = build_opener(MyHTTPHandler, MyOtherHTTPHandler) 1877 self.opener_has_handler(o, MyHTTPHandler) 1878 self.opener_has_handler(o, MyOtherHTTPHandler) 1879 1880 def test_HTTPError_interface(self): 1881 """ 1882 Issue 13211 reveals that HTTPError didn't implement the URLError 1883 interface even though HTTPError is a subclass of URLError. 1884 """ 1885 msg = 'something bad happened' 1886 url = code = fp = None 1887 hdrs = 'Content-Length: 42' 1888 err = urllib.error.HTTPError(url, code, msg, hdrs, fp) 1889 self.assertTrue(hasattr(err, 'reason')) 1890 self.assertEqual(err.reason, 'something bad happened') 1891 self.assertTrue(hasattr(err, 'headers')) 1892 self.assertEqual(err.headers, 'Content-Length: 42') 1893 expected_errmsg = 'HTTP Error %s: %s' % (err.code, err.msg) 1894 self.assertEqual(str(err), expected_errmsg) 1895 expected_errmsg = '<HTTPError %s: %r>' % (err.code, err.msg) 1896 self.assertEqual(repr(err), expected_errmsg) 1897 1898 def test_gh_98778(self): 1899 x = urllib.error.HTTPError("url", 405, "METHOD NOT ALLOWED", None, None) 1900 self.assertEqual(getattr(x, "__notes__", ()), ()) 1901 self.assertIsInstance(x.fp.read(), bytes) 1902 1903 def test_parse_proxy(self): 1904 parse_proxy_test_cases = [ 1905 ('proxy.example.com', 1906 (None, None, None, 'proxy.example.com')), 1907 ('proxy.example.com:3128', 1908 (None, None, None, 'proxy.example.com:3128')), 1909 ('proxy.example.com', (None, None, None, 'proxy.example.com')), 1910 ('proxy.example.com:3128', 1911 (None, None, None, 'proxy.example.com:3128')), 1912 # The authority component may optionally include userinfo 1913 # (assumed to be # username:password): 1914 ('joe:password@proxy.example.com', 1915 (None, 'joe', 'password', 'proxy.example.com')), 1916 ('joe:password@proxy.example.com:3128', 1917 (None, 'joe', 'password', 'proxy.example.com:3128')), 1918 #Examples with URLS 1919 ('http://proxy.example.com/', 1920 ('http', None, None, 'proxy.example.com')), 1921 ('http://proxy.example.com:3128/', 1922 ('http', None, None, 'proxy.example.com:3128')), 1923 ('http://joe:password@proxy.example.com/', 1924 ('http', 'joe', 'password', 'proxy.example.com')), 1925 ('http://joe:password@proxy.example.com:3128', 1926 ('http', 'joe', 'password', 'proxy.example.com:3128')), 1927 # Everything after the authority is ignored 1928 ('ftp://joe:password@proxy.example.com/rubbish:3128', 1929 ('ftp', 'joe', 'password', 'proxy.example.com')), 1930 # Test for no trailing '/' case 1931 ('http://joe:password@proxy.example.com', 1932 ('http', 'joe', 'password', 'proxy.example.com')), 1933 # Testcases with '/' character in username, password 1934 ('http://user/name:password@localhost:22', 1935 ('http', 'user/name', 'password', 'localhost:22')), 1936 ('http://username:pass/word@localhost:22', 1937 ('http', 'username', 'pass/word', 'localhost:22')), 1938 ('http://user/name:pass/word@localhost:22', 1939 ('http', 'user/name', 'pass/word', 'localhost:22')), 1940 ] 1941 1942 1943 for tc, expected in parse_proxy_test_cases: 1944 self.assertEqual(_parse_proxy(tc), expected) 1945 1946 self.assertRaises(ValueError, _parse_proxy, 'file:/ftp.example.com'), 1947 1948 def test_unsupported_algorithm(self): 1949 handler = AbstractDigestAuthHandler() 1950 with self.assertRaises(ValueError) as exc: 1951 handler.get_algorithm_impls('invalid') 1952 self.assertEqual( 1953 str(exc.exception), 1954 "Unsupported digest authentication algorithm 'invalid'" 1955 ) 1956 1957 1958class RequestTests(unittest.TestCase): 1959 class PutRequest(Request): 1960 method = 'PUT' 1961 1962 def setUp(self): 1963 self.get = Request("http://www.python.org/~jeremy/") 1964 self.post = Request("http://www.python.org/~jeremy/", 1965 "data", 1966 headers={"X-Test": "test"}) 1967 self.head = Request("http://www.python.org/~jeremy/", method='HEAD') 1968 self.put = self.PutRequest("http://www.python.org/~jeremy/") 1969 self.force_post = self.PutRequest("http://www.python.org/~jeremy/", 1970 method="POST") 1971 1972 def test_method(self): 1973 self.assertEqual("POST", self.post.get_method()) 1974 self.assertEqual("GET", self.get.get_method()) 1975 self.assertEqual("HEAD", self.head.get_method()) 1976 self.assertEqual("PUT", self.put.get_method()) 1977 self.assertEqual("POST", self.force_post.get_method()) 1978 1979 def test_data(self): 1980 self.assertFalse(self.get.data) 1981 self.assertEqual("GET", self.get.get_method()) 1982 self.get.data = "spam" 1983 self.assertTrue(self.get.data) 1984 self.assertEqual("POST", self.get.get_method()) 1985 1986 # issue 16464 1987 # if we change data we need to remove content-length header 1988 # (cause it's most probably calculated for previous value) 1989 def test_setting_data_should_remove_content_length(self): 1990 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1991 self.get.add_unredirected_header("Content-length", 42) 1992 self.assertEqual(42, self.get.unredirected_hdrs["Content-length"]) 1993 self.get.data = "spam" 1994 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1995 1996 # issue 17485 same for deleting data. 1997 def test_deleting_data_should_remove_content_length(self): 1998 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 1999 self.get.data = 'foo' 2000 self.get.add_unredirected_header("Content-length", 3) 2001 self.assertEqual(3, self.get.unredirected_hdrs["Content-length"]) 2002 del self.get.data 2003 self.assertNotIn("Content-length", self.get.unredirected_hdrs) 2004 2005 def test_get_full_url(self): 2006 self.assertEqual("http://www.python.org/~jeremy/", 2007 self.get.get_full_url()) 2008 2009 def test_selector(self): 2010 self.assertEqual("/~jeremy/", self.get.selector) 2011 req = Request("http://www.python.org/") 2012 self.assertEqual("/", req.selector) 2013 2014 def test_get_type(self): 2015 self.assertEqual("http", self.get.type) 2016 2017 def test_get_host(self): 2018 self.assertEqual("www.python.org", self.get.host) 2019 2020 def test_get_host_unquote(self): 2021 req = Request("http://www.%70ython.org/") 2022 self.assertEqual("www.python.org", req.host) 2023 2024 def test_proxy(self): 2025 self.assertFalse(self.get.has_proxy()) 2026 self.get.set_proxy("www.perl.org", "http") 2027 self.assertTrue(self.get.has_proxy()) 2028 self.assertEqual("www.python.org", self.get.origin_req_host) 2029 self.assertEqual("www.perl.org", self.get.host) 2030 2031 def test_wrapped_url(self): 2032 req = Request("<URL:http://www.python.org>") 2033 self.assertEqual("www.python.org", req.host) 2034 2035 def test_url_fragment(self): 2036 req = Request("http://www.python.org/?qs=query#fragment=true") 2037 self.assertEqual("/?qs=query", req.selector) 2038 req = Request("http://www.python.org/#fun=true") 2039 self.assertEqual("/", req.selector) 2040 2041 # Issue 11703: geturl() omits fragment in the original URL. 2042 url = 'http://docs.python.org/library/urllib2.html#OK' 2043 req = Request(url) 2044 self.assertEqual(req.get_full_url(), url) 2045 2046 def test_url_fullurl_get_full_url(self): 2047 urls = ['http://docs.python.org', 2048 'http://docs.python.org/library/urllib2.html#OK', 2049 'http://www.python.org/?qs=query#fragment=true'] 2050 for url in urls: 2051 req = Request(url) 2052 self.assertEqual(req.get_full_url(), req.full_url) 2053 2054 2055if __name__ == "__main__": 2056 unittest.main() 2057
