1# Copyright 2019 The gRPC Authors. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import asyncio 16import logging 17import time 18import unittest 19 20import grpc 21from grpc.experimental import aio 22 23from tests.unit import resources 24from tests.unit.framework.common import test_constants 25from tests_aio.unit._test_base import AioTestBase 26 27_SIMPLE_UNARY_UNARY = "/test/SimpleUnaryUnary" 28_BLOCK_FOREVER = "/test/BlockForever" 29_BLOCK_BRIEFLY = "/test/BlockBriefly" 30_UNARY_STREAM_ASYNC_GEN = "/test/UnaryStreamAsyncGen" 31_UNARY_STREAM_READER_WRITER = "/test/UnaryStreamReaderWriter" 32_UNARY_STREAM_EVILLY_MIXED = "/test/UnaryStreamEvillyMixed" 33_STREAM_UNARY_ASYNC_GEN = "/test/StreamUnaryAsyncGen" 34_STREAM_UNARY_READER_WRITER = "/test/StreamUnaryReaderWriter" 35_STREAM_UNARY_EVILLY_MIXED = "/test/StreamUnaryEvillyMixed" 36_STREAM_STREAM_ASYNC_GEN = "/test/StreamStreamAsyncGen" 37_STREAM_STREAM_READER_WRITER = "/test/StreamStreamReaderWriter" 38_STREAM_STREAM_EVILLY_MIXED = "/test/StreamStreamEvillyMixed" 39_UNIMPLEMENTED_METHOD = "/test/UnimplementedMethod" 40_ERROR_IN_STREAM_STREAM = "/test/ErrorInStreamStream" 41_ERROR_IN_STREAM_UNARY = "/test/ErrorInStreamUnary" 42_ERROR_WITHOUT_RAISE_IN_UNARY_UNARY = "/test/ErrorWithoutRaiseInUnaryUnary" 43_ERROR_WITHOUT_RAISE_IN_STREAM_STREAM = "/test/ErrorWithoutRaiseInStreamStream" 44_INVALID_TRAILING_METADATA = "/test/InvalidTrailingMetadata" 45 46_REQUEST = b"\x00\x00\x00" 47_RESPONSE = b"\x01\x01\x01" 48_NUM_STREAM_REQUESTS = 3 49_NUM_STREAM_RESPONSES = 5 50_MAXIMUM_CONCURRENT_RPCS = 5 51 52 53class _GenericHandler(grpc.GenericRpcHandler): 54 def __init__(self): 55 self._called = asyncio.get_event_loop().create_future() 56 self._routing_table = { 57 _SIMPLE_UNARY_UNARY: grpc.unary_unary_rpc_method_handler( 58 self._unary_unary 59 ), 60 _BLOCK_FOREVER: grpc.unary_unary_rpc_method_handler( 61 self._block_forever 62 ), 63 _BLOCK_BRIEFLY: grpc.unary_unary_rpc_method_handler( 64 self._block_briefly 65 ), 66 _UNARY_STREAM_ASYNC_GEN: grpc.unary_stream_rpc_method_handler( 67 self._unary_stream_async_gen 68 ), 69 _UNARY_STREAM_READER_WRITER: grpc.unary_stream_rpc_method_handler( 70 self._unary_stream_reader_writer 71 ), 72 _UNARY_STREAM_EVILLY_MIXED: grpc.unary_stream_rpc_method_handler( 73 self._unary_stream_evilly_mixed 74 ), 75 _STREAM_UNARY_ASYNC_GEN: grpc.stream_unary_rpc_method_handler( 76 self._stream_unary_async_gen 77 ), 78 _STREAM_UNARY_READER_WRITER: grpc.stream_unary_rpc_method_handler( 79 self._stream_unary_reader_writer 80 ), 81 _STREAM_UNARY_EVILLY_MIXED: grpc.stream_unary_rpc_method_handler( 82 self._stream_unary_evilly_mixed 83 ), 84 _STREAM_STREAM_ASYNC_GEN: grpc.stream_stream_rpc_method_handler( 85 self._stream_stream_async_gen 86 ), 87 _STREAM_STREAM_READER_WRITER: grpc.stream_stream_rpc_method_handler( 88 self._stream_stream_reader_writer 89 ), 90 _STREAM_STREAM_EVILLY_MIXED: grpc.stream_stream_rpc_method_handler( 91 self._stream_stream_evilly_mixed 92 ), 93 _ERROR_IN_STREAM_STREAM: grpc.stream_stream_rpc_method_handler( 94 self._error_in_stream_stream 95 ), 96 _ERROR_IN_STREAM_UNARY: grpc.stream_unary_rpc_method_handler( 97 self._value_error_in_stream_unary 98 ), 99 _ERROR_WITHOUT_RAISE_IN_UNARY_UNARY: grpc.unary_unary_rpc_method_handler( 100 self._error_without_raise_in_unary_unary 101 ), 102 _ERROR_WITHOUT_RAISE_IN_STREAM_STREAM: grpc.stream_stream_rpc_method_handler( 103 self._error_without_raise_in_stream_stream 104 ), 105 _INVALID_TRAILING_METADATA: grpc.unary_unary_rpc_method_handler( 106 self._invalid_trailing_metadata 107 ), 108 } 109 110 @staticmethod 111 async def _unary_unary(unused_request, unused_context): 112 return _RESPONSE 113 114 async def _block_forever(self, unused_request, unused_context): 115 await asyncio.get_event_loop().create_future() 116 117 async def _block_briefly(self, unused_request, unused_context): 118 await asyncio.sleep(test_constants.SHORT_TIMEOUT / 2) 119 return _RESPONSE 120 121 async def _unary_stream_async_gen(self, unused_request, unused_context): 122 for _ in range(_NUM_STREAM_RESPONSES): 123 yield _RESPONSE 124 125 async def _unary_stream_reader_writer(self, unused_request, context): 126 for _ in range(_NUM_STREAM_RESPONSES): 127 await context.write(_RESPONSE) 128 129 async def _unary_stream_evilly_mixed(self, unused_request, context): 130 yield _RESPONSE 131 for _ in range(_NUM_STREAM_RESPONSES - 1): 132 await context.write(_RESPONSE) 133 134 async def _stream_unary_async_gen(self, request_iterator, unused_context): 135 request_count = 0 136 async for request in request_iterator: 137 assert _REQUEST == request 138 request_count += 1 139 assert _NUM_STREAM_REQUESTS == request_count 140 return _RESPONSE 141 142 async def _stream_unary_reader_writer(self, unused_request, context): 143 for _ in range(_NUM_STREAM_REQUESTS): 144 assert _REQUEST == await context.read() 145 return _RESPONSE 146 147 async def _stream_unary_evilly_mixed(self, request_iterator, context): 148 assert _REQUEST == await context.read() 149 request_count = 0 150 async for request in request_iterator: 151 assert _REQUEST == request 152 request_count += 1 153 assert _NUM_STREAM_REQUESTS - 1 == request_count 154 return _RESPONSE 155 156 async def _stream_stream_async_gen(self, request_iterator, unused_context): 157 request_count = 0 158 async for request in request_iterator: 159 assert _REQUEST == request 160 request_count += 1 161 assert _NUM_STREAM_REQUESTS == request_count 162 163 for _ in range(_NUM_STREAM_RESPONSES): 164 yield _RESPONSE 165 166 async def _stream_stream_reader_writer(self, unused_request, context): 167 for _ in range(_NUM_STREAM_REQUESTS): 168 assert _REQUEST == await context.read() 169 for _ in range(_NUM_STREAM_RESPONSES): 170 await context.write(_RESPONSE) 171 172 async def _stream_stream_evilly_mixed(self, request_iterator, context): 173 assert _REQUEST == await context.read() 174 request_count = 0 175 async for request in request_iterator: 176 assert _REQUEST == request 177 request_count += 1 178 assert _NUM_STREAM_REQUESTS - 1 == request_count 179 180 yield _RESPONSE 181 for _ in range(_NUM_STREAM_RESPONSES - 1): 182 await context.write(_RESPONSE) 183 184 async def _error_in_stream_stream(self, request_iterator, unused_context): 185 async for request in request_iterator: 186 assert _REQUEST == request 187 raise RuntimeError("A testing RuntimeError!") 188 yield _RESPONSE 189 190 async def _value_error_in_stream_unary(self, request_iterator, context): 191 request_count = 0 192 async for request in request_iterator: 193 assert _REQUEST == request 194 request_count += 1 195 if request_count >= 1: 196 raise ValueError("A testing RuntimeError!") 197 198 async def _error_without_raise_in_unary_unary(self, request, context): 199 assert _REQUEST == request 200 context.set_code(grpc.StatusCode.INTERNAL) 201 202 async def _error_without_raise_in_stream_stream( 203 self, request_iterator, context 204 ): 205 async for request in request_iterator: 206 assert _REQUEST == request 207 context.set_code(grpc.StatusCode.INTERNAL) 208 209 async def _invalid_trailing_metadata(self, request, context): 210 assert _REQUEST == request 211 for invalid_metadata in [ 212 42, 213 {}, 214 {"error": "error"}, 215 [{"error": "error"}], 216 ]: 217 try: 218 context.set_trailing_metadata(invalid_metadata) 219 except TypeError: 220 pass 221 else: 222 raise ValueError( 223 "No TypeError raised for invalid metadata:" 224 f" {invalid_metadata}" 225 ) 226 227 await context.abort( 228 grpc.StatusCode.DATA_LOSS, 229 details="invalid abort", 230 trailing_metadata=({"error": ("error1", "error2")}), 231 ) 232 233 def service(self, handler_details): 234 if not self._called.done(): 235 self._called.set_result(None) 236 return self._routing_table.get(handler_details.method) 237 238 async def wait_for_call(self): 239 await self._called 240 241 242async def _start_test_server(): 243 server = aio.server() 244 port = server.add_insecure_port("[::]:0") 245 generic_handler = _GenericHandler() 246 server.add_generic_rpc_handlers((generic_handler,)) 247 await server.start() 248 return "localhost:%d" % port, server, generic_handler 249 250 251class TestServer(AioTestBase): 252 async def setUp(self): 253 addr, self._server, self._generic_handler = await _start_test_server() 254 self._channel = aio.insecure_channel(addr) 255 256 async def tearDown(self): 257 await self._channel.close() 258 await self._server.stop(None) 259 260 async def test_unary_unary(self): 261 unary_unary_call = self._channel.unary_unary(_SIMPLE_UNARY_UNARY) 262 response = await unary_unary_call(_REQUEST) 263 self.assertEqual(response, _RESPONSE) 264 265 async def test_unary_stream_async_generator(self): 266 unary_stream_call = self._channel.unary_stream(_UNARY_STREAM_ASYNC_GEN) 267 call = unary_stream_call(_REQUEST) 268 269 response_cnt = 0 270 async for response in call: 271 response_cnt += 1 272 self.assertEqual(_RESPONSE, response) 273 274 self.assertEqual(_NUM_STREAM_RESPONSES, response_cnt) 275 self.assertEqual(await call.code(), grpc.StatusCode.OK) 276 277 async def test_unary_stream_reader_writer(self): 278 unary_stream_call = self._channel.unary_stream( 279 _UNARY_STREAM_READER_WRITER 280 ) 281 call = unary_stream_call(_REQUEST) 282 283 for _ in range(_NUM_STREAM_RESPONSES): 284 response = await call.read() 285 self.assertEqual(_RESPONSE, response) 286 287 self.assertEqual(await call.code(), grpc.StatusCode.OK) 288 289 async def test_unary_stream_evilly_mixed(self): 290 unary_stream_call = self._channel.unary_stream( 291 _UNARY_STREAM_EVILLY_MIXED 292 ) 293 call = unary_stream_call(_REQUEST) 294 295 # Uses reader API 296 self.assertEqual(_RESPONSE, await call.read()) 297 298 # Uses async generator API, mixed! 299 with self.assertRaises(aio.UsageError): 300 async for response in call: 301 self.assertEqual(_RESPONSE, response) 302 303 async def test_stream_unary_async_generator(self): 304 stream_unary_call = self._channel.stream_unary(_STREAM_UNARY_ASYNC_GEN) 305 call = stream_unary_call() 306 307 for _ in range(_NUM_STREAM_REQUESTS): 308 await call.write(_REQUEST) 309 await call.done_writing() 310 311 response = await call 312 self.assertEqual(_RESPONSE, response) 313 self.assertEqual(await call.code(), grpc.StatusCode.OK) 314 315 async def test_stream_unary_async_generator_with_request_iter(self): 316 stream_unary_call = self._channel.stream_unary(_STREAM_UNARY_ASYNC_GEN) 317 318 finished = False 319 320 def request_gen(): 321 for _ in range(_NUM_STREAM_REQUESTS): 322 yield _REQUEST 323 nonlocal finished 324 finished = True 325 326 call = stream_unary_call(request_gen()) 327 328 response = await call 329 self.assertEqual(_RESPONSE, response) 330 self.assertEqual(await call.code(), grpc.StatusCode.OK) 331 self.assertEqual(finished, True) 332 333 async def test_stream_unary_reader_writer(self): 334 stream_unary_call = self._channel.stream_unary( 335 _STREAM_UNARY_READER_WRITER 336 ) 337 call = stream_unary_call() 338 339 for _ in range(_NUM_STREAM_REQUESTS): 340 await call.write(_REQUEST) 341 await call.done_writing() 342 343 response = await call 344 self.assertEqual(_RESPONSE, response) 345 self.assertEqual(await call.code(), grpc.StatusCode.OK) 346 347 async def test_stream_unary_evilly_mixed(self): 348 stream_unary_call = self._channel.stream_unary( 349 _STREAM_UNARY_EVILLY_MIXED 350 ) 351 call = stream_unary_call() 352 353 for _ in range(_NUM_STREAM_REQUESTS): 354 await call.write(_REQUEST) 355 await call.done_writing() 356 357 response = await call 358 self.assertEqual(_RESPONSE, response) 359 self.assertEqual(await call.code(), grpc.StatusCode.OK) 360 361 async def test_stream_stream_async_generator(self): 362 stream_stream_call = self._channel.stream_stream( 363 _STREAM_STREAM_ASYNC_GEN 364 ) 365 call = stream_stream_call() 366 367 for _ in range(_NUM_STREAM_REQUESTS): 368 await call.write(_REQUEST) 369 await call.done_writing() 370 371 for _ in range(_NUM_STREAM_RESPONSES): 372 response = await call.read() 373 self.assertEqual(_RESPONSE, response) 374 375 self.assertEqual(await call.code(), grpc.StatusCode.OK) 376 377 async def test_stream_stream_reader_writer(self): 378 stream_stream_call = self._channel.stream_stream( 379 _STREAM_STREAM_READER_WRITER 380 ) 381 call = stream_stream_call() 382 383 for _ in range(_NUM_STREAM_REQUESTS): 384 await call.write(_REQUEST) 385 await call.done_writing() 386 387 for _ in range(_NUM_STREAM_RESPONSES): 388 response = await call.read() 389 self.assertEqual(_RESPONSE, response) 390 391 self.assertEqual(await call.code(), grpc.StatusCode.OK) 392 393 async def test_stream_stream_evilly_mixed(self): 394 stream_stream_call = self._channel.stream_stream( 395 _STREAM_STREAM_EVILLY_MIXED 396 ) 397 call = stream_stream_call() 398 399 for _ in range(_NUM_STREAM_REQUESTS): 400 await call.write(_REQUEST) 401 await call.done_writing() 402 403 for _ in range(_NUM_STREAM_RESPONSES): 404 response = await call.read() 405 self.assertEqual(_RESPONSE, response) 406 407 self.assertEqual(await call.code(), grpc.StatusCode.OK) 408 409 async def test_shutdown(self): 410 await self._server.stop(None) 411 # Ensures no SIGSEGV triggered, and ends within timeout. 412 413 async def test_shutdown_after_call(self): 414 await self._channel.unary_unary(_SIMPLE_UNARY_UNARY)(_REQUEST) 415 416 await self._server.stop(None) 417 418 async def test_graceful_shutdown_success(self): 419 call = self._channel.unary_unary(_BLOCK_BRIEFLY)(_REQUEST) 420 await self._generic_handler.wait_for_call() 421 422 shutdown_start_time = time.time() 423 await self._server.stop(test_constants.SHORT_TIMEOUT) 424 grace_period_length = time.time() - shutdown_start_time 425 self.assertGreater( 426 grace_period_length, test_constants.SHORT_TIMEOUT / 3 427 ) 428 429 # Validates the states. 430 self.assertEqual(_RESPONSE, await call) 431 self.assertTrue(call.done()) 432 433 async def test_graceful_shutdown_failed(self): 434 call = self._channel.unary_unary(_BLOCK_FOREVER)(_REQUEST) 435 await self._generic_handler.wait_for_call() 436 437 await self._server.stop(test_constants.SHORT_TIMEOUT) 438 439 with self.assertRaises(aio.AioRpcError) as exception_context: 440 await call 441 self.assertEqual( 442 grpc.StatusCode.UNAVAILABLE, exception_context.exception.code() 443 ) 444 445 async def test_concurrent_graceful_shutdown(self): 446 call = self._channel.unary_unary(_BLOCK_BRIEFLY)(_REQUEST) 447 await self._generic_handler.wait_for_call() 448 449 # Expects the shortest grace period to be effective. 450 shutdown_start_time = time.time() 451 await asyncio.gather( 452 self._server.stop(test_constants.LONG_TIMEOUT), 453 self._server.stop(test_constants.SHORT_TIMEOUT), 454 self._server.stop(test_constants.LONG_TIMEOUT), 455 ) 456 grace_period_length = time.time() - shutdown_start_time 457 self.assertGreater( 458 grace_period_length, test_constants.SHORT_TIMEOUT / 3 459 ) 460 461 self.assertEqual(_RESPONSE, await call) 462 self.assertTrue(call.done()) 463 464 async def test_concurrent_graceful_shutdown_immediate(self): 465 call = self._channel.unary_unary(_BLOCK_FOREVER)(_REQUEST) 466 await self._generic_handler.wait_for_call() 467 468 # Expects no grace period, due to the "server.stop(None)". 469 await asyncio.gather( 470 self._server.stop(test_constants.LONG_TIMEOUT), 471 self._server.stop(None), 472 self._server.stop(test_constants.SHORT_TIMEOUT), 473 self._server.stop(test_constants.LONG_TIMEOUT), 474 ) 475 476 with self.assertRaises(aio.AioRpcError) as exception_context: 477 await call 478 self.assertEqual( 479 grpc.StatusCode.UNAVAILABLE, exception_context.exception.code() 480 ) 481 482 async def test_shutdown_before_call(self): 483 await self._server.stop(None) 484 485 # Ensures the server is cleaned up at this point. 486 # Some proper exception should be raised. 487 with self.assertRaises(aio.AioRpcError): 488 await self._channel.unary_unary(_SIMPLE_UNARY_UNARY)(_REQUEST) 489 490 async def test_unimplemented(self): 491 call = self._channel.unary_unary(_UNIMPLEMENTED_METHOD) 492 with self.assertRaises(aio.AioRpcError) as exception_context: 493 await call(_REQUEST) 494 rpc_error = exception_context.exception 495 self.assertEqual(grpc.StatusCode.UNIMPLEMENTED, rpc_error.code()) 496 497 async def test_shutdown_during_stream_stream(self): 498 stream_stream_call = self._channel.stream_stream( 499 _STREAM_STREAM_ASYNC_GEN 500 ) 501 call = stream_stream_call() 502 503 # Don't half close the RPC yet, keep it alive. 504 await call.write(_REQUEST) 505 await self._server.stop(None) 506 507 self.assertEqual(grpc.StatusCode.UNAVAILABLE, await call.code()) 508 # No segfault 509 510 async def test_error_in_stream_stream(self): 511 stream_stream_call = self._channel.stream_stream( 512 _ERROR_IN_STREAM_STREAM 513 ) 514 call = stream_stream_call() 515 516 # Don't half close the RPC yet, keep it alive. 517 await call.write(_REQUEST) 518 519 # Don't segfault here 520 self.assertEqual(grpc.StatusCode.UNKNOWN, await call.code()) 521 522 async def test_error_without_raise_in_unary_unary(self): 523 call = self._channel.unary_unary(_ERROR_WITHOUT_RAISE_IN_UNARY_UNARY)( 524 _REQUEST 525 ) 526 527 with self.assertRaises(aio.AioRpcError) as exception_context: 528 await call 529 530 rpc_error = exception_context.exception 531 self.assertEqual(grpc.StatusCode.INTERNAL, rpc_error.code()) 532 533 async def test_error_without_raise_in_stream_stream(self): 534 call = self._channel.stream_stream( 535 _ERROR_WITHOUT_RAISE_IN_STREAM_STREAM 536 )() 537 538 for _ in range(_NUM_STREAM_REQUESTS): 539 await call.write(_REQUEST) 540 await call.done_writing() 541 542 self.assertEqual(grpc.StatusCode.INTERNAL, await call.code()) 543 544 async def test_error_in_stream_unary(self): 545 stream_unary_call = self._channel.stream_unary(_ERROR_IN_STREAM_UNARY) 546 547 async def request_gen(): 548 for _ in range(_NUM_STREAM_REQUESTS): 549 yield _REQUEST 550 551 call = stream_unary_call(request_gen()) 552 553 with self.assertRaises(aio.AioRpcError) as exception_context: 554 await call 555 rpc_error = exception_context.exception 556 self.assertEqual(grpc.StatusCode.UNKNOWN, rpc_error.code()) 557 558 async def test_port_binding_exception(self): 559 server = aio.server(options=(("grpc.so_reuseport", 0),)) 560 port = server.add_insecure_port("localhost:0") 561 bind_address = "localhost:%d" % port 562 563 with self.assertRaises(RuntimeError): 564 server.add_insecure_port(bind_address) 565 566 server_credentials = grpc.ssl_server_credentials( 567 [(resources.private_key(), resources.certificate_chain())] 568 ) 569 with self.assertRaises(RuntimeError): 570 server.add_secure_port(bind_address, server_credentials) 571 572 async def test_maximum_concurrent_rpcs(self): 573 async def coro_wrapper(awaitable): 574 return await awaitable 575 576 # Build the server with concurrent rpc argument 577 server = aio.server(maximum_concurrent_rpcs=_MAXIMUM_CONCURRENT_RPCS) 578 port = server.add_insecure_port("localhost:0") 579 bind_address = "localhost:%d" % port 580 server.add_generic_rpc_handlers((_GenericHandler(),)) 581 await server.start() 582 # Build the channel 583 channel = aio.insecure_channel(bind_address) 584 # Deplete the concurrent quota with 3 times of max RPCs 585 rpc_tasks = [] 586 for _ in range(3 * _MAXIMUM_CONCURRENT_RPCS): 587 task = asyncio.create_task( 588 coro_wrapper(channel.unary_unary(_BLOCK_BRIEFLY)(_REQUEST)) 589 ) 590 rpc_tasks.append(task) 591 await_tasks = asyncio.wait(rpc_tasks, return_when=asyncio.ALL_COMPLETED) 592 593 done, _ = await await_tasks 594 exceptions = [] 595 for task in done: 596 exception = task.exception() 597 if exception: 598 exceptions.append(exception) 599 600 # Check whether the number of tasks raised RESOURCE_EXHAUSTED is roughly 601 # 2 * _MAXIMUM_CONCURRENT_RPCS. 602 self.assertAlmostEqual( 603 len(exceptions), 604 2 * _MAXIMUM_CONCURRENT_RPCS, 605 delta=_MAXIMUM_CONCURRENT_RPCS, 606 ) 607 for exception in exceptions: 608 self.assertTrue(isinstance(exception, aio.AioRpcError)) 609 self.assertEqual( 610 grpc.StatusCode.RESOURCE_EXHAUSTED, exception.code() 611 ) 612 self.assertIn("Concurrent RPC limit exceeded", exception.details()) 613 614 # Clean-up 615 await channel.close() 616 await server.stop(0) 617 618 async def test_invalid_trailing_metadata(self): 619 call = self._channel.unary_unary(_INVALID_TRAILING_METADATA)(_REQUEST) 620 621 with self.assertRaises(aio.AioRpcError) as exception_context: 622 await call 623 624 rpc_error = exception_context.exception 625 self.assertEqual(grpc.StatusCode.UNKNOWN, rpc_error.code()) 626 self.assertIn("trailing", rpc_error.details()) 627 628 629if __name__ == "__main__": 630 logging.basicConfig(level=logging.DEBUG) 631 unittest.main(verbosity=2) 632